diff --git a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs index 8979c2a19..026d0aae5 100644 --- a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs +++ b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs @@ -67,7 +67,7 @@ namespace NzbDrone.Common.Test.Http res = _httpClient.GetAsync($"https://{site}/status/429").GetAwaiter().GetResult(); - if (res == null || res.StatusCode != (HttpStatusCode)429) + if (res == null || res.StatusCode != HttpStatusCode.TooManyRequests) { return false; } diff --git a/src/NzbDrone.Common/Http/HttpClient.cs b/src/NzbDrone.Common/Http/HttpClient.cs index 6fb27fbb3..b0c3acb13 100644 --- a/src/NzbDrone.Common/Http/HttpClient.cs +++ b/src/NzbDrone.Common/Http/HttpClient.cs @@ -118,7 +118,7 @@ namespace NzbDrone.Common.Http _logger.Warn("HTTP Error - {0}", response); } - if ((int)response.StatusCode == 429) + if (response.StatusCode == HttpStatusCode.TooManyRequests) { throw new TooManyRequestsException(request, response); } diff --git a/src/NzbDrone.Core/Http/CloudFlare/CloudFlareCaptchaException.cs b/src/NzbDrone.Core/Http/CloudFlare/CloudFlareCaptchaException.cs deleted file mode 100644 index db88d5c7b..000000000 --- a/src/NzbDrone.Core/Http/CloudFlare/CloudFlareCaptchaException.cs +++ /dev/null @@ -1,21 +0,0 @@ -using NzbDrone.Common.Exceptions; -using NzbDrone.Common.Http; - -namespace NzbDrone.Core.Http.CloudFlare -{ - public class CloudFlareCaptchaException : NzbDroneException - { - public HttpResponse Response { get; set; } - - public CloudFlareCaptchaRequest CaptchaRequest { get; set; } - - public CloudFlareCaptchaException(HttpResponse response, CloudFlareCaptchaRequest captchaRequest) - : base("Unable to access {0}, blocked by CloudFlare CAPTCHA. Likely due to shared-IP VPN.", response.Request.Url.Host) - { - Response = response; - CaptchaRequest = captchaRequest; - } - - public bool IsExpired => Response.Request.Cookies.ContainsKey("cf_clearance"); - } -} diff --git a/src/NzbDrone.Core/Http/CloudFlare/CloudFlareCaptchaRequest.cs b/src/NzbDrone.Core/Http/CloudFlare/CloudFlareCaptchaRequest.cs deleted file mode 100644 index 332a01059..000000000 --- a/src/NzbDrone.Core/Http/CloudFlare/CloudFlareCaptchaRequest.cs +++ /dev/null @@ -1,15 +0,0 @@ -using NzbDrone.Common.Http; - -namespace NzbDrone.Core.Http.CloudFlare -{ - public class CloudFlareCaptchaRequest - { - public string Host { get; set; } - public string SiteKey { get; set; } - - public string Ray { get; set; } - public string SecretToken { get; set; } - - public HttpUri ResponseUrl { get; set; } - } -} diff --git a/src/NzbDrone.Core/Http/CloudFlare/CloudFlareDetectionService.cs b/src/NzbDrone.Core/Http/CloudFlare/CloudFlareDetectionService.cs new file mode 100644 index 000000000..e00838b67 --- /dev/null +++ b/src/NzbDrone.Core/Http/CloudFlare/CloudFlareDetectionService.cs @@ -0,0 +1,45 @@ +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text.RegularExpressions; +using NLog; +using NzbDrone.Common.Http; + +namespace NzbDrone.Core.Http.CloudFlare +{ + public class CloudFlareDetectionService + { + private static readonly HashSet CloudflareServerNames = new HashSet { "cloudflare", "cloudflare-nginx", "ddos-guard" }; + private readonly Logger _logger; + + public CloudFlareDetectionService(Logger logger) + { + _logger = logger; + } + + public static bool IsCloudflareProtected(HttpResponse response) + { + if (!response.Headers.Any(i => i.Key != null && i.Key.ToLower() == "server" && CloudflareServerNames.Contains(i.Value.ToLower()))) + { + return false; + } + + // detect CloudFlare and DDoS-GUARD + if (response.StatusCode.Equals(HttpStatusCode.ServiceUnavailable) || + response.StatusCode.Equals(HttpStatusCode.Forbidden)) + { + return true; // Defected CloudFlare and DDoS-GUARD + } + + // detect Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands + if (response.Headers.Vary.ToString() == "Accept-Encoding,User-Agent" && + response.Headers.ContentEncoding.ToString() == "" && + response.Content.ToLower().Contains("ddos")) + { + return true; + } + + return false; + } + } +} diff --git a/src/NzbDrone.Core/Http/CloudFlare/CloudFlareHttpInterceptor.cs b/src/NzbDrone.Core/Http/CloudFlare/CloudFlareHttpInterceptor.cs deleted file mode 100644 index dca1d12c0..000000000 --- a/src/NzbDrone.Core/Http/CloudFlare/CloudFlareHttpInterceptor.cs +++ /dev/null @@ -1,54 +0,0 @@ -using System.Net; -using System.Text.RegularExpressions; -using NLog; -using NzbDrone.Common.Http; - -namespace NzbDrone.Core.Http.CloudFlare -{ - public class CloudFlareHttpInterceptor : IHttpRequestInterceptor - { - private const string _cloudFlareChallengeScript = "cdn-cgi/scripts/cf.challenge.js"; - private readonly Logger _logger; - private static readonly Regex _cloudFlareRegex = new Regex(@"data-ray=""(?[\w-_]+)"".*?data-sitekey=""(?[\w-_]+)"".*?data-stoken=""(?[\w-_]+)""", RegexOptions.Compiled); - - public CloudFlareHttpInterceptor(Logger logger) - { - _logger = logger; - } - - public HttpRequest PreRequest(HttpRequest request) - { - return request; - } - - public HttpResponse PostResponse(HttpResponse response) - { - if (response.StatusCode == HttpStatusCode.Forbidden && response.Content.Contains(_cloudFlareChallengeScript)) - { - _logger.Debug("CloudFlare CAPTCHA block on {0}", response.Request.Url); - throw new CloudFlareCaptchaException(response, CreateCaptchaRequest(response)); - } - - return response; - } - - private CloudFlareCaptchaRequest CreateCaptchaRequest(HttpResponse response) - { - var match = _cloudFlareRegex.Match(response.Content); - - if (!match.Success) - { - return null; - } - - return new CloudFlareCaptchaRequest - { - Host = response.Request.Url.Host, - SiteKey = match.Groups["SiteKey"].Value, - Ray = match.Groups["Ray"].Value, - SecretToken = match.Groups["SecretToken"].Value, - ResponseUrl = response.Request.Url + new HttpUri("/cdn-cgi/l/chk_captcha") - }; - } - } -} diff --git a/src/NzbDrone.Core/Http/CloudFlare/CloudFlareProtectionException.cs b/src/NzbDrone.Core/Http/CloudFlare/CloudFlareProtectionException.cs new file mode 100644 index 000000000..7ebcd4863 --- /dev/null +++ b/src/NzbDrone.Core/Http/CloudFlare/CloudFlareProtectionException.cs @@ -0,0 +1,16 @@ +using NzbDrone.Common.Exceptions; +using NzbDrone.Common.Http; + +namespace NzbDrone.Core.Http.CloudFlare +{ + public class CloudFlareProtectionException : NzbDroneException + { + public HttpResponse Response { get; set; } + + public CloudFlareProtectionException(HttpResponse response) + : base("Unable to access {0}, blocked by CloudFlare Protection.", response.Request.Url.Host) + { + Response = response; + } + } +} diff --git a/src/NzbDrone.Core/IndexerProxies/FlareSolverr/FlareSolverr.cs b/src/NzbDrone.Core/IndexerProxies/FlareSolverr/FlareSolverr.cs index 0275b7178..ff54e3fc9 100644 --- a/src/NzbDrone.Core/IndexerProxies/FlareSolverr/FlareSolverr.cs +++ b/src/NzbDrone.Core/IndexerProxies/FlareSolverr/FlareSolverr.cs @@ -11,6 +11,7 @@ using NzbDrone.Common.Cloud; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Common.Serializer; +using NzbDrone.Core.Http.CloudFlare; using NzbDrone.Core.Localization; using NzbDrone.Core.Validation; @@ -45,7 +46,7 @@ namespace NzbDrone.Core.IndexerProxies.FlareSolverr public override HttpResponse PostResponse(HttpResponse response) { - if (!IsCloudflareProtected(response)) + if (!CloudFlareDetectionService.IsCloudflareProtected(response)) { _logger.Debug("CF Protection not detected, returning original response"); return response; @@ -53,14 +54,12 @@ namespace NzbDrone.Core.IndexerProxies.FlareSolverr var flaresolverrResponse = _httpClient.Execute(GenerateFlareSolverrRequest(response.Request)); - FlareSolverrResponse result = null; - if (flaresolverrResponse.StatusCode != HttpStatusCode.OK && flaresolverrResponse.StatusCode != HttpStatusCode.InternalServerError) { throw new FlareSolverrException("HTTP StatusCode not 200 or 500. Status is :" + response.StatusCode); } - result = JsonConvert.DeserializeObject(flaresolverrResponse.Content); + var result = JsonConvert.DeserializeObject(flaresolverrResponse.Content); var newRequest = response.Request; @@ -76,31 +75,6 @@ namespace NzbDrone.Core.IndexerProxies.FlareSolverr return finalResponse; } - private static bool IsCloudflareProtected(HttpResponse response) - { - if (!response.Headers.Any(i => i.Key != null && i.Key.ToLower() == "server" && CloudflareServerNames.Contains(i.Value.ToLower()))) - { - return false; - } - - // detect CloudFlare and DDoS-GUARD - if (response.StatusCode.Equals(HttpStatusCode.ServiceUnavailable) || - response.StatusCode.Equals(HttpStatusCode.Forbidden)) - { - return true; // Defected CloudFlare and DDoS-GUARD - } - - // detect Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands - if (response.Headers.Vary.ToString() == "Accept-Encoding,User-Agent" && - response.Headers.ContentEncoding.ToString() == "" && - response.Content.ToLower().Contains("ddos")) - { - return true; - } - - return false; - } - private void InjectCookies(HttpRequest request, FlareSolverrResponse flareSolverrResponse) { var rCookies = flareSolverrResponse.Solution.Cookies; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Anidub.cs b/src/NzbDrone.Core/Indexers/Definitions/Anidub.cs index 323942b75..08b918aa1 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Anidub.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Anidub.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Collections.Specialized; using System.Globalization; using System.Linq; +using System.Net; using System.Net.Http; using System.Text; using System.Text.RegularExpressions; @@ -493,7 +494,7 @@ namespace NzbDrone.Core.Indexers.Definitions // Throw common http errors here before we try to parse if (releaseResponse.HttpResponse.HasHttpError) { - if ((int)releaseResponse.HttpResponse.StatusCode == 429) + if (releaseResponse.HttpResponse.StatusCode == HttpStatusCode.TooManyRequests) { throw new TooManyRequestsException(releaseRequest.HttpRequest, releaseResponse.HttpResponse); } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Animedia.cs b/src/NzbDrone.Core/Indexers/Definitions/Animedia.cs index 3b73777a8..0710550ba 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Animedia.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Animedia.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Collections.Specialized; +using System.Net; using System.Text; using System.Text.RegularExpressions; using AngleSharp.Html.Parser; @@ -307,7 +308,7 @@ namespace NzbDrone.Core.Indexers.Definitions // Throw common http errors here before we try to parse if (releaseResponse.HttpResponse.HasHttpError) { - if ((int)releaseResponse.HttpResponse.StatusCode == 429) + if (releaseResponse.HttpResponse.StatusCode == HttpStatusCode.TooManyRequests) { throw new TooManyRequestsException(releaseRequest.HttpRequest, releaseResponse.HttpResponse); } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs index 23285b6c3..34d0f8a99 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs @@ -194,7 +194,7 @@ namespace NzbDrone.Core.Indexers.Cardigann throw new ReleaseUnavailableException("Downloading torrent failed", ex); } - if ((int)ex.Response.StatusCode == 429) + if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) { _logger.Error("API Grab Limit reached for {0}", request.Url.FullUri); } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs index 4bb424ec9..4fc04fbf4 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs @@ -101,29 +101,12 @@ namespace NzbDrone.Core.Indexers.Rarbg { Settings.Validate().Filter("BaseUrl").ThrowOnError(); - try - { - var request = new HttpRequestBuilder(Settings.BaseUrl.Trim('/')) + var request = new HttpRequestBuilder(Settings.BaseUrl.Trim('/')) .Resource($"/pubapi_v2.php?get_token=get_token&app_id={BuildInfo.AppName}") .Accept(HttpAccept.Json) .Build(); - _httpClient.Get(request); - } - catch (CloudFlareCaptchaException ex) - { - return new - { - captchaRequest = new - { - host = ex.CaptchaRequest.Host, - ray = ex.CaptchaRequest.Ray, - siteKey = ex.CaptchaRequest.SiteKey, - secretToken = ex.CaptchaRequest.SecretToken, - responseUrl = ex.CaptchaRequest.ResponseUrl.FullUri, - } - }; - } + _httpClient.Get(request); return new { diff --git a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgRequestGenerator.cs index 29093f389..cef7b90f0 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgRequestGenerator.cs @@ -26,12 +26,6 @@ namespace NzbDrone.Core.Indexers.Rarbg .Resource("/pubapi_v2.php") .Accept(HttpAccept.Json); - if (Settings.CaptchaToken.IsNotNullOrWhiteSpace()) - { - requestBuilder.UseSimplifiedUserAgent = true; - requestBuilder.SetCookie("cf_clearance", Settings.CaptchaToken); - } - requestBuilder.AddQueryParam("mode", "search"); if (imdbId.IsNotNullOrWhiteSpace()) diff --git a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgSettings.cs b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgSettings.cs index 6eef9727a..45c091e6d 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgSettings.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgSettings.cs @@ -14,8 +14,5 @@ namespace NzbDrone.Core.Indexers.Rarbg [FieldDefinition(2, Type = FieldType.Checkbox, Label = "Ranked Only", HelpText = "Only include ranked results.")] public bool RankedOnly { get; set; } - - [FieldDefinition(3, Type = FieldType.Captcha, Label = "CAPTCHA Token", HelpText = "CAPTCHA Clearance token used to handle CloudFlare Anti-DDOS measures on shared-ip VPNs.")] - public string CaptchaToken { get; set; } } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgTokenProvider.cs b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgTokenProvider.cs index 613aa1f39..60069d089 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgTokenProvider.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgTokenProvider.cs @@ -36,12 +36,6 @@ namespace NzbDrone.Core.Indexers.Rarbg .Resource($"/pubapi_v2.php?get_token=get_token&app_id={BuildInfo.AppName}") .Accept(HttpAccept.Json); - if (settings.CaptchaToken.IsNotNullOrWhiteSpace()) - { - requestBuilder.UseSimplifiedUserAgent = true; - requestBuilder.SetCookie("cf_clearance", settings.CaptchaToken); - } - var response = _httpClient.Get(requestBuilder.Build()); return response.Resource["token"].ToString(); diff --git a/src/NzbDrone.Core/Indexers/Definitions/SpeedApp.cs b/src/NzbDrone.Core/Indexers/Definitions/SpeedApp.cs index 2bd83027b..d4b696246 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/SpeedApp.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/SpeedApp.cs @@ -154,7 +154,7 @@ namespace NzbDrone.Core.Indexers.Definitions throw new ReleaseUnavailableException("Downloading torrent failed", ex); } - if ((int)ex.Response.StatusCode == 429) + if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) { _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); } diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index 0e0d435fb..9adc525e2 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -232,7 +232,7 @@ namespace NzbDrone.Core.Indexers _indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1)); } - _logger.Warn("API Request Limit reached for {0}", this); + _logger.Warn("Request Limit reached for {0}", this); } catch (HttpException ex) { @@ -251,19 +251,12 @@ namespace NzbDrone.Core.Indexers _indexerStatusService.RecordFailure(Definition.Id); _logger.Warn("Invalid Credentials for {0} {1}", this, url); } - catch (CloudFlareCaptchaException ex) + catch (CloudFlareProtectionException ex) { result.Queries.Add(new IndexerQueryResult { Response = ex.Response }); _indexerStatusService.RecordFailure(Definition.Id); ex.WithData("FeedUrl", url); - if (ex.IsExpired) - { - _logger.Error(ex, "Expired CAPTCHA token for {0}, please refresh in indexer settings.", this); - } - else - { - _logger.Error(ex, "CAPTCHA token required for {0}, check indexer settings.", this); - } + _logger.Error(ex, "Cloudflare protection detected for {0}, Flaresolverr may be required.", this); } catch (IndexerException ex) { @@ -399,12 +392,17 @@ namespace NzbDrone.Core.Indexers { _logger.Warn("HTTP Error - {0}", response); - if ((int)response.StatusCode == 429) + if (response.StatusCode == HttpStatusCode.TooManyRequests) { throw new TooManyRequestsException(request.HttpRequest, response); } } + if (CloudFlareDetectionService.IsCloudflareProtected(response)) + { + throw new CloudFlareProtectionException(response); + } + UpdateCookies(Cookies, DateTime.Now + TimeSpan.FromDays(30)); return new IndexerResponse(request, response); @@ -471,16 +469,9 @@ namespace NzbDrone.Core.Indexers { _logger.Warn("Request limit reached: " + ex.Message); } - catch (CloudFlareCaptchaException ex) + catch (CloudFlareProtectionException ex) { - if (ex.IsExpired) - { - return new ValidationFailure("CaptchaToken", "CloudFlare CAPTCHA token expired, please Refresh."); - } - else - { - return new ValidationFailure("CaptchaToken", "Site protected by CloudFlare CAPTCHA. Valid CAPTCHA token required."); - } + return new ValidationFailure(string.Empty, ex.Message); } catch (UnsupportedFeedException ex) { diff --git a/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs b/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs index 451d001a7..b804c2903 100644 --- a/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs @@ -54,7 +54,7 @@ namespace NzbDrone.Core.Indexers throw new ReleaseUnavailableException("Downloading torrent failed", ex); } - if ((int)ex.Response.StatusCode == 429) + if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) { _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); } diff --git a/src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs b/src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs index af3044c33..f8104a844 100644 --- a/src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs @@ -52,7 +52,7 @@ namespace NzbDrone.Core.Indexers throw new ReleaseUnavailableException("Downloading nzb failed", ex); } - if ((int)ex.Response.StatusCode == 429) + if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) { _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); }