diff --git a/src/Jackett.Common/Exceptions/TooManyRequestsException.cs b/src/Jackett.Common/Exceptions/TooManyRequestsException.cs new file mode 100644 index 000000000..d6a5b3113 --- /dev/null +++ b/src/Jackett.Common/Exceptions/TooManyRequestsException.cs @@ -0,0 +1,27 @@ +using System; +using Jackett.Common.Utils.Clients; + +namespace Jackett.Common.Exceptions +{ + public class TooManyRequestsException : Exception + { + public TimeSpan RetryAfter { get; private set; } + + public TooManyRequestsException(string message, TimeSpan retryWait) + : base(message) => RetryAfter = retryWait; + + public TooManyRequestsException(string message, WebResult response) + : base(message) + { + if (response.Headers.ContainsKey("Retry-After")) + { + var retryAfter = response.Headers["Retry-After"].ToString(); + + if (int.TryParse(retryAfter, out var seconds)) + RetryAfter = TimeSpan.FromSeconds(seconds); + else if (DateTime.TryParse(retryAfter, out var date)) + RetryAfter = date.ToUniversalTime() - DateTime.UtcNow; + } + } + } +} diff --git a/src/Jackett.Common/IndexerException.cs b/src/Jackett.Common/IndexerException.cs index bd4f607a6..bbf115398 100644 --- a/src/Jackett.Common/IndexerException.cs +++ b/src/Jackett.Common/IndexerException.cs @@ -7,17 +7,17 @@ namespace Jackett.Common { public IIndexer Indexer { get; protected set; } - public IndexerException(IIndexer Indexer, string message, Exception innerException) + public IndexerException(IIndexer indexer, string message, Exception innerException) : base(message, innerException) - => this.Indexer = Indexer; + => this.Indexer = indexer; - public IndexerException(IIndexer Indexer, string message) - : this(Indexer, message, null) + public IndexerException(IIndexer indexer, string message) + : this(indexer, message, null) { } - public IndexerException(IIndexer Indexer, Exception innerException) - : this(Indexer, "Exception (" + Indexer.Id + "): " + innerException.GetBaseException().Message, innerException) + public IndexerException(IIndexer indexer, Exception innerException) + : this(indexer, "Exception (" + indexer.Id + "): " + innerException.GetBaseException().Message, innerException) { } } diff --git a/src/Jackett.Common/Indexers/BaseIndexer.cs b/src/Jackett.Common/Indexers/BaseIndexer.cs index 4405aece6..b13c61b00 100644 --- a/src/Jackett.Common/Indexers/BaseIndexer.cs +++ b/src/Jackett.Common/Indexers/BaseIndexer.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; +using Jackett.Common.Exceptions; using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; @@ -315,6 +316,12 @@ namespace Jackett.Common.Indexers expireAt = DateTime.Now.Add(HealthyStatusValidity); return new IndexerResult(this, results, false); } + catch (TooManyRequestsException ex) + { + var delay = ex.RetryAfter.TotalSeconds; + expireAt = DateTime.Now.AddSeconds(delay); + throw new IndexerException(this, ex); + } catch (Exception ex) { var delay = Math.Min(MaxStatusValidity.TotalSeconds, ErrorStatusValidity.TotalSeconds * Math.Pow(2, errorCount++)); diff --git a/src/Jackett.Common/Indexers/RarBG.cs b/src/Jackett.Common/Indexers/RarBG.cs index 440305a74..86d597a2d 100644 --- a/src/Jackett.Common/Indexers/RarBG.cs +++ b/src/Jackett.Common/Indexers/RarBG.cs @@ -6,8 +6,8 @@ using System.Globalization; using System.Linq; using System.Net; using System.Text; -using System.Threading; using System.Threading.Tasks; +using Jackett.Common.Exceptions; using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; @@ -139,25 +139,20 @@ namespace Jackett.Common.Indexers await RenewalTokenAsync(); var response = await RequestWithCookiesAsync(BuildSearchUrl(query)); - if (response != null && response.ContentString.StartsWith("<")) + var responseCode = (int)response.Status; + + switch (responseCode) { - if (response.ContentString.Contains("torrentapi.org | 520:")) - { - if (retry) - { - logger.Warn("torrentapi.org returned Error 520, retrying after 5 secs"); - return await PerformQueryWithRetry(query, false); - } - else - { - logger.Warn("torrentapi.org returned Error 520"); - return releases; - } - } - // the response was not JSON, likely a HTML page for a server outage - logger.Warn(response.ContentString); - throw new Exception("The response was not JSON"); + case 429: + throw new TooManyRequestsException($"Rate limited with StatusCode {responseCode}, retry in 2 minutes", TimeSpan.FromMinutes(2)); + case 520: + throw new TooManyRequestsException($"Rate limited with StatusCode {responseCode}, retry in 3 minutes", TimeSpan.FromMinutes(3)); + case (int)HttpStatusCode.OK: + break; + default: + throw new Exception($"Indexer API call returned an unexpected StatusCode [{responseCode}]"); } + var jsonContent = JObject.Parse(response.ContentString); var errorCode = jsonContent.Value("error_code"); switch (errorCode) @@ -176,11 +171,8 @@ namespace Jackett.Common.Indexers logger.Warn("torrentapi.org returned code 5 Too many requests per second, retrying after 5 secs"); return await PerformQueryWithRetry(query, false); } - else - { - logger.Warn("torrentapi.org returned code 5 Too many requests per second"); - return releases; - } + + throw new TooManyRequestsException("Rate limited, retry in 2 minutes", TimeSpan.FromMinutes(2)); case 8: // search_imdb not found, see issue #12466 (no longer used, has been replaced with error 10) case 9: // invalid imdb, see Radarr #1845 case 13: // invalid tmdb, invalid tvdb @@ -188,32 +180,23 @@ namespace Jackett.Common.Indexers case 10: // imdb not found, see issue #1486 case 14: // tmdb not found (see Radarr #7625), thetvdb not found case 20: // no results found - if (jsonContent.ContainsKey("rate_limit")) - { - if (retry) - { - logger.Warn("torrentapi.org returned code 20 with Rate Limit exceeded. Retrying after 5 secs."); - return await PerformQueryWithRetry(query, false); - } - else - { - logger.Warn("torrentapi.org returned code 20 with Rate Limit exceeded."); - return releases; - } - } + if (jsonContent.Value("rate_limit") is 1 && jsonContent.Value("torrent_results") == null) + throw new TooManyRequestsException("Rate limited, retry in 5 minutes", TimeSpan.FromMinutes(5)); + // the api returns "no results" in some valid queries. we do one retry on this case but we can't do more // because we can't distinguish between search without results and api malfunction return retry ? await PerformQueryWithRetry(query, false) : releases; default: - throw new Exception("Unknown error code: " + errorCode + " response: " + response.ContentString); + throw new Exception($"Unknown error code: {errorCode}. Response: {response.ContentString}"); } + if (jsonContent.Value("torrent_results") == null) + return releases; + try { foreach (var item in jsonContent.Value("torrent_results")) { - var title = WebUtility.HtmlDecode(item.Value("title")); - var magnetStr = item.Value("download"); var magnetUri = new Uri(magnetStr); @@ -224,27 +207,21 @@ namespace Jackett.Common.Indexers // append app_id to prevent api server returning 403 forbidden var details = new Uri(item.Value("info_page") + "&app_id=" + _appId); - // ex: 2015-08-16 21:25:08 +0000 - var dateStr = item.Value("pubdate").Replace(" +0000", ""); - var dateTime = DateTime.ParseExact(dateStr, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); - var publishDate = DateTime.SpecifyKind(dateTime, DateTimeKind.Utc).ToLocalTime(); - - var size = item.Value("size"); var seeders = item.Value("seeders"); var leechers = item.Value("leechers"); var release = new ReleaseInfo { - Title = title, - Category = MapTrackerCatDescToNewznab(item.Value("category")), - MagnetUri = magnetUri, - InfoHash = infoHash, - Details = details, - PublishDate = publishDate, Guid = guid, + Details = details, + MagnetUri = magnetUri, + Title = WebUtility.HtmlDecode(item.Value("title")).Trim(), + Category = MapTrackerCatDescToNewznab(item.Value("category")), + InfoHash = infoHash, + PublishDate = DateTime.Parse(item.Value("pubdate"), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal), Seeders = seeders, Peers = leechers + seeders, - Size = size, + Size = item.Value("size"), DownloadVolumeFactor = 0, UploadVolumeFactor = 1 }; diff --git a/src/Jackett.Server/Controllers/ResultsController.cs b/src/Jackett.Server/Controllers/ResultsController.cs index 888614ae7..a7b429bbd 100644 --- a/src/Jackett.Server/Controllers/ResultsController.cs +++ b/src/Jackett.Server/Controllers/ResultsController.cs @@ -7,6 +7,7 @@ using System.Text; using System.Threading.Tasks; using System.Xml.Linq; using Jackett.Common; +using Jackett.Common.Exceptions; using Jackett.Common.Indexers; using Jackett.Common.Indexers.Meta; using Jackett.Common.Models; @@ -14,9 +15,11 @@ using Jackett.Common.Models.DTO; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.Filters; using Microsoft.AspNetCore.Routing; +using Microsoft.Net.Http.Headers; using NLog; namespace Jackett.Server.Controllers @@ -432,6 +435,20 @@ namespace Jackett.Server.Controllers return Content(xml, "application/rss+xml", Encoding.UTF8); } + catch (IndexerException ex) when (ex.InnerException is TooManyRequestsException tooManyRequestsException) + { + logger.Error(ex); + + if (!HttpContext.Response.Headers.ContainsKey("Retry-After")) + { + var retryAfter = Convert.ToInt32(tooManyRequestsException.RetryAfter.TotalSeconds); + + if (retryAfter > 0) + HttpContext.Response.Headers.Add("Retry-After", $"{retryAfter}"); + } + + return GetErrorXML(900, ex.Message, StatusCodes.Status429TooManyRequests); + } catch (Exception e) { logger.Error(e); @@ -440,7 +457,18 @@ namespace Jackett.Server.Controllers } [Route("[action]/{ignored?}")] - public IActionResult GetErrorXML(int code, string description) => Content(CreateErrorXML(code, description), "application/xml", Encoding.UTF8); + public IActionResult GetErrorXML(int code, string description, int statusCode = StatusCodes.Status400BadRequest) + { + var mediaTypeHeaderValue = MediaTypeHeaderValue.Parse("application/xml"); + mediaTypeHeaderValue.Encoding = Encoding.UTF8; + + return new ContentResult + { + StatusCode = statusCode, + Content = CreateErrorXML(code, description), + ContentType = mediaTypeHeaderValue.ToString() + }; + } public static string CreateErrorXML(int code, string description) {