shpaq/Jackett-Jackett
1
0
Fork 0
mirror of https://github.com/Jackett/Jackett.git synced 2025-09-17 17:34:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

rarbg: implement retry strategy with 429 response (#14000)

Browse Source
This commit is contained in:
Bogdan
2023-02-19 13:26:49 +02:00
committed by GitHub
parent 3adf750973
commit 2321c14584
5 changed files with 98 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/Jackett.Common/Exceptions/TooManyRequestsException.cs Normal file
View File

@@ -0,0 +1,27 @@
using System;
using Jackett.Common.Utils.Clients;
namespace Jackett.Common.Exceptions
{
public class TooManyRequestsException : Exception
{
public TimeSpan RetryAfter { get; private set; }
public TooManyRequestsException(string message, TimeSpan retryWait)
: base(message) => RetryAfter = retryWait;
public TooManyRequestsException(string message, WebResult response)
: base(message)
{
if (response.Headers.ContainsKey("Retry-After"))
{
var retryAfter = response.Headers["Retry-After"].ToString();
if (int.TryParse(retryAfter, out var seconds))
RetryAfter = TimeSpan.FromSeconds(seconds);
else if (DateTime.TryParse(retryAfter, out var date))
RetryAfter = date.ToUniversalTime() - DateTime.UtcNow;
}
}
}
}

12
src/Jackett.Common/IndexerException.cs
View File

@@ -7,17 +7,17 @@ namespace Jackett.Common
{
public IIndexer Indexer { get; protected set; }
public IndexerException(IIndexer Indexer, string message, Exception innerException)
public IndexerException(IIndexer indexer, string message, Exception innerException)
: base(message, innerException)
=> this.Indexer = Indexer;
=> this.Indexer = indexer;
public IndexerException(IIndexer Indexer, string message)
: this(Indexer, message, null)
public IndexerException(IIndexer indexer, string message)
: this(indexer, message, null)
{
}
public IndexerException(IIndexer Indexer, Exception innerException)
: this(Indexer, "Exception (" + Indexer.Id + "): " + innerException.GetBaseException().Message, innerException)
public IndexerException(IIndexer indexer, Exception innerException)
: this(indexer, "Exception (" + indexer.Id + "): " + innerException.GetBaseException().Message, innerException)
{
}
}

7
src/Jackett.Common/Indexers/BaseIndexer.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Jackett.Common.Exceptions;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
@@ -315,6 +316,12 @@ namespace Jackett.Common.Indexers
expireAt = DateTime.Now.Add(HealthyStatusValidity);
return new IndexerResult(this, results, false);
}
catch (TooManyRequestsException ex)
{
var delay = ex.RetryAfter.TotalSeconds;
expireAt = DateTime.Now.AddSeconds(delay);
throw new IndexerException(this, ex);
}
catch (Exception ex)
{
var delay = Math.Min(MaxStatusValidity.TotalSeconds, ErrorStatusValidity.TotalSeconds * Math.Pow(2, errorCount++));

81
src/Jackett.Common/Indexers/RarBG.cs
View File

@@ -6,8 +6,8 @@ using System.Globalization;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Jackett.Common.Exceptions;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
@@ -139,25 +139,20 @@ namespace Jackett.Common.Indexers
await RenewalTokenAsync();
var response = await RequestWithCookiesAsync(BuildSearchUrl(query));
if (response != null && response.ContentString.StartsWith("<"))
var responseCode = (int)response.Status;
switch (responseCode)
{
if (response.ContentString.Contains("torrentapi.org | 520:"))
{
if (retry)
{
logger.Warn("torrentapi.org returned Error 520, retrying after 5 secs");
return await PerformQueryWithRetry(query, false);
}
else
{
logger.Warn("torrentapi.org returned Error 520");
return releases;
}
}
// the response was not JSON, likely a HTML page for a server outage
logger.Warn(response.ContentString);
throw new Exception("The response was not JSON");
case 429:
throw new TooManyRequestsException($"Rate limited with StatusCode {responseCode}, retry in 2 minutes", TimeSpan.FromMinutes(2));
case 520:
throw new TooManyRequestsException($"Rate limited with StatusCode {responseCode}, retry in 3 minutes", TimeSpan.FromMinutes(3));
case (int)HttpStatusCode.OK:
break;
default:
throw new Exception($"Indexer API call returned an unexpected StatusCode [{responseCode}]");
}
var jsonContent = JObject.Parse(response.ContentString);
var errorCode = jsonContent.Value<int>("error_code");
switch (errorCode)
@@ -176,11 +171,8 @@ namespace Jackett.Common.Indexers
logger.Warn("torrentapi.org returned code 5 Too many requests per second, retrying after 5 secs");
return await PerformQueryWithRetry(query, false);
}
else
{
logger.Warn("torrentapi.org returned code 5 Too many requests per second");
return releases;
}
throw new TooManyRequestsException("Rate limited, retry in 2 minutes", TimeSpan.FromMinutes(2));
case 8: // search_imdb not found, see issue #12466 (no longer used, has been replaced with error 10)
case 9: // invalid imdb, see Radarr #1845
case 13: // invalid tmdb, invalid tvdb
@@ -188,32 +180,23 @@ namespace Jackett.Common.Indexers
case 10: // imdb not found, see issue #1486
case 14: // tmdb not found (see Radarr #7625), thetvdb not found
case 20: // no results found
if (jsonContent.ContainsKey("rate_limit"))
{
if (retry)
{
logger.Warn("torrentapi.org returned code 20 with Rate Limit exceeded. Retrying after 5 secs.");
return await PerformQueryWithRetry(query, false);
}
else
{
logger.Warn("torrentapi.org returned code 20 with Rate Limit exceeded.");
return releases;
}
}
if (jsonContent.Value<int>("rate_limit") is 1 && jsonContent.Value<JArray>("torrent_results") == null)
throw new TooManyRequestsException("Rate limited, retry in 5 minutes", TimeSpan.FromMinutes(5));
// the api returns "no results" in some valid queries. we do one retry on this case but we can't do more
// because we can't distinguish between search without results and api malfunction
return retry ? await PerformQueryWithRetry(query, false) : releases;
default:
throw new Exception("Unknown error code: " + errorCode + " response: " + response.ContentString);
throw new Exception($"Unknown error code: {errorCode}. Response: {response.ContentString}");
}
if (jsonContent.Value<JArray>("torrent_results") == null)
return releases;
try
{
foreach (var item in jsonContent.Value<JArray>("torrent_results"))
{
var title = WebUtility.HtmlDecode(item.Value<string>("title"));
var magnetStr = item.Value<string>("download");
var magnetUri = new Uri(magnetStr);
@@ -224,27 +207,21 @@ namespace Jackett.Common.Indexers
// append app_id to prevent api server returning 403 forbidden
var details = new Uri(item.Value<string>("info_page") + "&app_id=" + _appId);
// ex: 2015-08-16 21:25:08 +0000
var dateStr = item.Value<string>("pubdate").Replace(" +0000", "");
var dateTime = DateTime.ParseExact(dateStr, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
var publishDate = DateTime.SpecifyKind(dateTime, DateTimeKind.Utc).ToLocalTime();
var size = item.Value<long>("size");
var seeders = item.Value<int>("seeders");
var leechers = item.Value<int>("leechers");
var release = new ReleaseInfo
{
Title = title,
Category = MapTrackerCatDescToNewznab(item.Value<string>("category")),
MagnetUri = magnetUri,
InfoHash = infoHash,
Details = details,
PublishDate = publishDate,
Guid = guid,
Details = details,
MagnetUri = magnetUri,
Title = WebUtility.HtmlDecode(item.Value<string>("title")).Trim(),
Category = MapTrackerCatDescToNewznab(item.Value<string>("category")),
InfoHash = infoHash,
PublishDate = DateTime.Parse(item.Value<string>("pubdate"), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal),
Seeders = seeders,
Peers = leechers + seeders,
Size = size,
Size = item.Value<long>("size"),
DownloadVolumeFactor = 0,
UploadVolumeFactor = 1
};

30
src/Jackett.Server/Controllers/ResultsController.cs
View File

@@ -7,6 +7,7 @@ using System.Text;
using System.Threading.Tasks;
using System.Xml.Linq;
using Jackett.Common;
using Jackett.Common.Exceptions;
using Jackett.Common.Indexers;
using Jackett.Common.Indexers.Meta;
using Jackett.Common.Models;
@@ -14,9 +15,11 @@ using Jackett.Common.Models.DTO;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.Filters;
using Microsoft.AspNetCore.Routing;
using Microsoft.Net.Http.Headers;
using NLog;
namespace Jackett.Server.Controllers
@@ -432,6 +435,20 @@ namespace Jackett.Server.Controllers
return Content(xml, "application/rss+xml", Encoding.UTF8);
}
catch (IndexerException ex) when (ex.InnerException is TooManyRequestsException tooManyRequestsException)
{
logger.Error(ex);
if (!HttpContext.Response.Headers.ContainsKey("Retry-After"))
{
var retryAfter = Convert.ToInt32(tooManyRequestsException.RetryAfter.TotalSeconds);
if (retryAfter > 0)
HttpContext.Response.Headers.Add("Retry-After", $"{retryAfter}");
}
return GetErrorXML(900, ex.Message, StatusCodes.Status429TooManyRequests);
}
catch (Exception e)
{
logger.Error(e);
@@ -440,7 +457,18 @@ namespace Jackett.Server.Controllers
}
[Route("[action]/{ignored?}")]
public IActionResult GetErrorXML(int code, string description) => Content(CreateErrorXML(code, description), "application/xml", Encoding.UTF8);
public IActionResult GetErrorXML(int code, string description, int statusCode = StatusCodes.Status400BadRequest)
{
var mediaTypeHeaderValue = MediaTypeHeaderValue.Parse("application/xml");
mediaTypeHeaderValue.Encoding = Encoding.UTF8;
return new ContentResult
{
StatusCode = statusCode,
Content = CreateErrorXML(code, description),
ContentType = mediaTypeHeaderValue.ToString()
};
}
public static string CreateErrorXML(int code, string description)
{