From 5cb13e69d7df1342e07bdc009f65f3b9b5f7606a Mon Sep 17 00:00:00 2001 From: Qstick Date: Sun, 16 May 2021 23:21:29 -0400 Subject: [PATCH] Fixed: Indexer failures not logged to history --- .../Http/Dispatchers/ManagedHttpDispatcher.cs | 8 +++++++- src/NzbDrone.Common/Http/HttpResponse.cs | 9 ++++++--- .../IndexerSearch/NzbSearchService.cs | 2 +- .../BroadcastheNet/BroadcastheNetParser.cs | 2 +- .../Headphones/HeadphonesRssParser.cs | 2 +- .../Definitions/Newznab/NewznabRssParser.cs | 2 +- .../Definitions/Torznab/TorznabRssParser.cs | 2 +- .../RequestLimitReachedException.cs | 12 ++++++++--- src/NzbDrone.Core/Indexers/HttpIndexerBase.cs | 20 +++++++++---------- src/NzbDrone.Core/Indexers/IndexerResponse.cs | 4 +--- 10 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs index 079db2707..d51a14929 100644 --- a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs +++ b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs @@ -68,6 +68,10 @@ namespace NzbDrone.Common.Http.Dispatchers HttpWebResponse httpWebResponse; + var sw = new Stopwatch(); + + sw.Start(); + try { if (request.ContentData != null) @@ -144,7 +148,9 @@ namespace NzbDrone.Common.Http.Dispatchers } } - return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), httpWebResponse.Cookies, data, httpWebResponse.StatusCode); + sw.Stop(); + + return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), httpWebResponse.Cookies, data, sw.ElapsedMilliseconds, httpWebResponse.StatusCode); } public async Task DownloadFileAsync(string url, string fileName) diff --git a/src/NzbDrone.Common/Http/HttpResponse.cs b/src/NzbDrone.Common/Http/HttpResponse.cs index 85ebe9ffc..9969c2bdc 100644 --- a/src/NzbDrone.Common/Http/HttpResponse.cs +++ b/src/NzbDrone.Common/Http/HttpResponse.cs @@ -11,16 +11,17 @@ namespace NzbDrone.Common.Http { private static readonly Regex RegexSetCookie = new Regex("^(.*?)=(.*?)(?:;|$)", RegexOptions.Compiled); - public HttpResponse(HttpRequest request, HttpHeader headers, CookieCollection cookies, byte[] binaryData, HttpStatusCode statusCode = HttpStatusCode.OK) + public HttpResponse(HttpRequest request, HttpHeader headers, CookieCollection cookies, byte[] binaryData, long elapsedTime = 0, HttpStatusCode statusCode = HttpStatusCode.OK) { Request = request; Headers = headers; Cookies = cookies; ResponseData = binaryData; StatusCode = statusCode; + ElapsedTime = elapsedTime; } - public HttpResponse(HttpRequest request, HttpHeader headers, CookieCollection cookies, string content, HttpStatusCode statusCode = HttpStatusCode.OK) + public HttpResponse(HttpRequest request, HttpHeader headers, CookieCollection cookies, string content, long elapsedTime = 0, HttpStatusCode statusCode = HttpStatusCode.OK) { Request = request; Headers = headers; @@ -28,12 +29,14 @@ namespace NzbDrone.Common.Http ResponseData = Headers.GetEncodingFromContentType().GetBytes(content); _content = content; StatusCode = statusCode; + ElapsedTime = elapsedTime; } public HttpRequest Request { get; private set; } public HttpHeader Headers { get; private set; } public CookieCollection Cookies { get; private set; } public HttpStatusCode StatusCode { get; private set; } + public long ElapsedTime { get; private set; } public byte[] ResponseData { get; private set; } private string _content; @@ -93,7 +96,7 @@ namespace NzbDrone.Common.Http where T : new() { public HttpResponse(HttpResponse response) - : base(response.Request, response.Headers, response.Cookies, response.ResponseData, response.StatusCode) + : base(response.Request, response.Headers, response.Cookies, response.ResponseData, response.ElapsedTime, response.StatusCode) { Resource = Json.Deserialize(response.Content); } diff --git a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs index f1221261a..dfce8f9b5 100644 --- a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs +++ b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs @@ -167,7 +167,7 @@ namespace NzbDrone.Core.IndexerSearch foreach (var query in indexerReports.Queries) { - _eventAggregator.PublishEvent(new IndexerQueryEvent(indexer.Definition.Id, criteriaBase, query.ElapsedTime, true, indexerReports.Releases.Count())); + _eventAggregator.PublishEvent(new IndexerQueryEvent(indexer.Definition.Id, criteriaBase, query.ElapsedTime, query.StatusCode == 200, indexerReports.Releases.Count())); } return indexerReports.Releases; diff --git a/src/NzbDrone.Core/Indexers/Definitions/BroadcastheNet/BroadcastheNetParser.cs b/src/NzbDrone.Core/Indexers/Definitions/BroadcastheNet/BroadcastheNetParser.cs index 741c86cbf..d75777c04 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/BroadcastheNet/BroadcastheNetParser.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/BroadcastheNet/BroadcastheNetParser.cs @@ -33,7 +33,7 @@ namespace NzbDrone.Core.Indexers.BroadcastheNet case HttpStatusCode.NotFound: throw new IndexerException(indexerResponse, "Indexer API call returned NotFound, the Indexer API may have changed."); case HttpStatusCode.ServiceUnavailable: - throw new RequestLimitReachedException("Cannot do more than 150 API requests per hour."); + throw new RequestLimitReachedException(indexerResponse, "Cannot do more than 150 API requests per hour."); default: if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) { diff --git a/src/NzbDrone.Core/Indexers/Definitions/Headphones/HeadphonesRssParser.cs b/src/NzbDrone.Core/Indexers/Definitions/Headphones/HeadphonesRssParser.cs index 757fb19e0..d1becd0b2 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Headphones/HeadphonesRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Headphones/HeadphonesRssParser.cs @@ -46,7 +46,7 @@ namespace NzbDrone.Core.Indexers.Headphones if (errorMessage == "Request limit reached") { - throw new RequestLimitReachedException("API limit reached"); + throw new RequestLimitReachedException(indexerResponse, "API limit reached"); } throw new NewznabException(indexerResponse, errorMessage); diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs index 53b62359b..4ad03e97a 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs @@ -45,7 +45,7 @@ namespace NzbDrone.Core.Indexers.Newznab if (errorMessage == "Request limit reached") { - throw new RequestLimitReachedException("API limit reached"); + throw new RequestLimitReachedException(indexerResponse, "API limit reached"); } throw new NewznabException(indexerResponse, errorMessage); diff --git a/src/NzbDrone.Core/Indexers/Definitions/Torznab/TorznabRssParser.cs b/src/NzbDrone.Core/Indexers/Definitions/Torznab/TorznabRssParser.cs index 7da495d9b..917c80104 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Torznab/TorznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Torznab/TorznabRssParser.cs @@ -42,7 +42,7 @@ namespace NzbDrone.Core.Indexers.Torznab if (errorMessage == "Request limit reached") { - throw new RequestLimitReachedException("API limit reached"); + throw new RequestLimitReachedException(indexerResponse, "API limit reached"); } throw new TorznabException("Torznab error detected: {0}", errorMessage); diff --git a/src/NzbDrone.Core/Indexers/Exceptions/RequestLimitReachedException.cs b/src/NzbDrone.Core/Indexers/Exceptions/RequestLimitReachedException.cs index 2ad6ecd81..b408dc162 100644 --- a/src/NzbDrone.Core/Indexers/Exceptions/RequestLimitReachedException.cs +++ b/src/NzbDrone.Core/Indexers/Exceptions/RequestLimitReachedException.cs @@ -1,17 +1,23 @@ -using NzbDrone.Common.Exceptions; +using NzbDrone.Common.Exceptions; namespace NzbDrone.Core.Indexers.Exceptions { public class RequestLimitReachedException : NzbDroneException { - public RequestLimitReachedException(string message, params object[] args) + private readonly IndexerResponse _indexerResponse; + + public RequestLimitReachedException(IndexerResponse response, string message, params object[] args) : base(message, args) { + _indexerResponse = response; } - public RequestLimitReachedException(string message) + public RequestLimitReachedException(IndexerResponse response, string message) : base(message) { + _indexerResponse = response; } + + public IndexerResponse Response => _indexerResponse; } } diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index 18e8c965c..91d695a15 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -302,6 +302,8 @@ namespace NzbDrone.Core.Indexers } catch (TooManyRequestsException ex) { + result.Queries.Add(new IndexerQueryResult { ElapsedTime = ex.Response.ElapsedTime, StatusCode = (int)ex.Response.StatusCode }); + if (ex.RetryAfter != TimeSpan.Zero) { _indexerStatusService.RecordFailure(Definition.Id, ex.RetryAfter); @@ -315,11 +317,13 @@ namespace NzbDrone.Core.Indexers } catch (HttpException ex) { + result.Queries.Add(new IndexerQueryResult { ElapsedTime = ex.Response.ElapsedTime, StatusCode = (int)ex.Response.StatusCode }); _indexerStatusService.RecordFailure(Definition.Id); _logger.Warn("{0} {1}", this, ex.Message); } - catch (RequestLimitReachedException) + catch (RequestLimitReachedException ex) { + result.Queries.Add(new IndexerQueryResult { ElapsedTime = ex.Response.HttpResponse.ElapsedTime, StatusCode = (int)ex.Response.HttpResponse.StatusCode }); _indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1)); _logger.Warn("API Request Limit reached for {0}", this); } @@ -330,6 +334,7 @@ namespace NzbDrone.Core.Indexers } catch (CloudFlareCaptchaException ex) { + result.Queries.Add(new IndexerQueryResult { ElapsedTime = ex.Response.ElapsedTime, StatusCode = (int)ex.Response.StatusCode }); _indexerStatusService.RecordFailure(Definition.Id); ex.WithData("FeedUrl", url); if (ex.IsExpired) @@ -343,6 +348,7 @@ namespace NzbDrone.Core.Indexers } catch (IndexerException ex) { + result.Queries.Add(new IndexerQueryResult { ElapsedTime = ex.Response.HttpResponse.ElapsedTime, StatusCode = (int)ex.Response.HttpResponse.StatusCode }); _indexerStatusService.RecordFailure(Definition.Id); _logger.Warn(ex, "{0}", url); } @@ -384,7 +390,7 @@ namespace NzbDrone.Core.Indexers return new IndexerQueryResult { Releases = releases, - ElapsedTime = response.ElapsedTime, + ElapsedTime = response.HttpResponse.ElapsedTime, StatusCode = (int)response.HttpResponse.StatusCode }; } @@ -450,13 +456,9 @@ namespace NzbDrone.Core.Indexers } } - var stopWatch = Stopwatch.StartNew(); - request.HttpRequest.SuppressHttpError = true; var response = await _httpClient.ExecuteAsync(request.HttpRequest); - stopWatch.Stop(); - // Check reponse to see if auth is needed, if needed try again if (CheckIfLoginNeeded(response)) { @@ -486,16 +488,14 @@ namespace NzbDrone.Core.Indexers UpdateCookies(Cookies, DateTime.Now + TimeSpan.FromDays(30)); - return new IndexerResponse(request, response, stopWatch.ElapsedMilliseconds); + return new IndexerResponse(request, response); } protected async Task ExecuteAuth(HttpRequest request) { - var stopWatch = Stopwatch.StartNew(); var response = await _httpClient.ExecuteAsync(request); - stopWatch.Stop(); - _eventAggregator.PublishEvent(new IndexerAuthEvent(Definition.Id, !response.HasHttpError, stopWatch.ElapsedMilliseconds)); + _eventAggregator.PublishEvent(new IndexerAuthEvent(Definition.Id, !response.HasHttpError, response.ElapsedTime)); return response; } diff --git a/src/NzbDrone.Core/Indexers/IndexerResponse.cs b/src/NzbDrone.Core/Indexers/IndexerResponse.cs index d8ddba910..a2bbc3f89 100644 --- a/src/NzbDrone.Core/Indexers/IndexerResponse.cs +++ b/src/NzbDrone.Core/Indexers/IndexerResponse.cs @@ -7,11 +7,10 @@ namespace NzbDrone.Core.Indexers private readonly IndexerRequest _indexerRequest; private readonly HttpResponse _httpResponse; - public IndexerResponse(IndexerRequest indexerRequest, HttpResponse httpResponse, long elapsedTime = 0) + public IndexerResponse(IndexerRequest indexerRequest, HttpResponse httpResponse) { _indexerRequest = indexerRequest; _httpResponse = httpResponse; - ElapsedTime = elapsedTime; } public IndexerRequest Request => _indexerRequest; @@ -21,6 +20,5 @@ namespace NzbDrone.Core.Indexers public HttpResponse HttpResponse => _httpResponse; public string Content => _httpResponse.Content; - public long ElapsedTime { get; private set; } } }