From 72ef99deace821663b426c4a666f4e76accb816d Mon Sep 17 00:00:00 2001
From: 6cUbi57z <3359745+6cUbi57z@users.noreply.github.com>
Date: Mon, 15 Mar 2021 06:27:18 +0000
Subject: [PATCH] Add retry logic to Anidex (#11318)
---
src/Jackett.Common/Indexers/Anidex.cs | 2 +
src/Jackett.Common/Indexers/BaseIndexer.cs | 105 +++++++++++++++++----
src/Jackett.Common/Indexers/RarBG.cs | 2 +
src/Jackett.Common/Jackett.Common.csproj | 1 +
4 files changed, 91 insertions(+), 19 deletions(-)
diff --git a/src/Jackett.Common/Indexers/Anidex.cs b/src/Jackett.Common/Indexers/Anidex.cs
index 8d2d5efe9..2d4abfe61 100644
--- a/src/Jackett.Common/Indexers/Anidex.cs
+++ b/src/Jackett.Common/Indexers/Anidex.cs
@@ -121,6 +121,8 @@ namespace Jackett.Common.Indexers
})
{ Name = "Order", Value = "desc" };
configData.AddDynamic("orderrequestedfromsite", orderSelect);
+
+ EnableConfigurableRetryAttempts();
}
private string GetLang => ((SelectItem)configData.GetDynamic("languageid")).Value;
diff --git a/src/Jackett.Common/Indexers/BaseIndexer.cs b/src/Jackett.Common/Indexers/BaseIndexer.cs
index f0172734e..801c27f47 100644
--- a/src/Jackett.Common/Indexers/BaseIndexer.cs
+++ b/src/Jackett.Common/Indexers/BaseIndexer.cs
@@ -9,6 +9,7 @@ using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
+using Polly;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
@@ -410,6 +411,88 @@ namespace Jackett.Common.Indexers
IProtectionService p, ICacheService cacheService)
: base("/", "", "", "", configService, logger, null, p, cacheService) => webclient = client;
+ protected virtual int DefaultNumberOfRetryAttempts => 2;
+
+ ///
+ /// Number of retry attempts to make if a web request fails.
+ ///
+ ///
+ /// Number of retries can be overridden for unstable indexers by overriding this property. Note that retry attempts include an
+ /// exponentially increasing delay.
+ ///
+ /// Alternatively, can be called in the constructor to add user configurable options.
+ ///
+ protected virtual int NumberOfRetryAttempts
+ {
+ get
+ {
+ var configItem = configData.GetDynamic("retryAttempts");
+ if (configItem == null)
+ {
+ // No config specified so use the default.
+ return DefaultNumberOfRetryAttempts;
+ }
+
+ var configValue = ((SelectItem)configItem).Value;
+
+ if (int.TryParse(configValue, out int parsedConfigValue) && parsedConfigValue > 0)
+ {
+ return parsedConfigValue;
+ }
+ else
+ {
+ // No config specified so use the default.
+ return DefaultNumberOfRetryAttempts;
+ }
+ }
+ }
+
+ private AsyncPolicy RetryPolicy
+ {
+ get
+ {
+ // Configure the retry policy
+ int attemptNumber = 1;
+ var retryPolicy = Policy
+ .HandleResult(r => (int)r.Status >= 500)
+ .Or()
+ .WaitAndRetryAsync(
+ NumberOfRetryAttempts,
+ retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt) / 4),
+ onRetry: (exception, timeSpan, context) =>
+ {
+ logger.Warn($"Request to {DisplayName} failed with status {exception.Result.Status}. Retrying in {timeSpan.TotalSeconds}s... (Attempt {attemptNumber} of {NumberOfRetryAttempts}).");
+ attemptNumber++;
+ });
+ return retryPolicy;
+ }
+ }
+
+ ///
+ /// Adds configuration options to allow the user to manually configure request retries.
+ ///
+ ///
+ /// This should only be enabled for indexers known to be unstable. To control the default value, override .
+ ///
+ protected void EnableConfigurableRetryAttempts()
+ {
+ var attemptSelect = new SelectItem(
+ new Dictionary
+ {
+ {"0", "No retries (fail fast)"},
+ {"1", "1 retry (0.5s delay)"},
+ {"2", "2 retries (1s delay)"},
+ {"3", "3 retries (2s delay)"},
+ {"4", "4 retries (4s delay)"},
+ {"5", "5 retries (8s delay)"}
+ })
+ {
+ Name = "Number of retries",
+ Value = DefaultNumberOfRetryAttempts.ToString()
+ };
+ configData.AddDynamic("retryAttempts", attemptSelect);
+ }
+
public virtual async Task Download(Uri link)
{
var uncleanLink = UncleanLink(link);
@@ -449,25 +532,9 @@ namespace Jackett.Common.Indexers
string referer = null, IEnumerable> data = null,
Dictionary headers = null, string rawbody = null, bool? emulateBrowser = null)
{
- Exception lastException = null;
- for (var i = 0; i < 3; i++)
- {
- try
- {
- return await RequestWithCookiesAsync(
- url, cookieOverride, method, referer, data, headers, rawbody, emulateBrowser);
- }
- catch (Exception e)
- {
- logger.Error(
- e, string.Format("On attempt {0} downloading from {1}: {2}", (i + 1), DisplayName, e.Message));
- lastException = e;
- }
-
- await Task.Delay(500);
- }
-
- throw lastException;
+ return await RetryPolicy.ExecuteAsync(async () =>
+ await RequestWithCookiesAsync(url, cookieOverride, method, referer, data, headers, rawbody, emulateBrowser)
+ );
}
protected virtual async Task RequestWithCookiesAsync(
diff --git a/src/Jackett.Common/Indexers/RarBG.cs b/src/Jackett.Common/Indexers/RarBG.cs
index d1bce483a..1f2a72b37 100644
--- a/src/Jackett.Common/Indexers/RarBG.cs
+++ b/src/Jackett.Common/Indexers/RarBG.cs
@@ -108,6 +108,8 @@ namespace Jackett.Common.Indexers
AddCategoryMapping(54, TorznabCatType.MoviesHD, "Movies/x265/1080");
_appId = "jackett_" + EnvironmentUtil.JackettVersion();
+
+ EnableConfigurableRetryAttempts();
}
public override void LoadValuesFromJson(JToken jsonConfig, bool useProtectionService = false)
diff --git a/src/Jackett.Common/Jackett.Common.csproj b/src/Jackett.Common/Jackett.Common.csproj
index ebffb41ac..22f17f3e0 100644
--- a/src/Jackett.Common/Jackett.Common.csproj
+++ b/src/Jackett.Common/Jackett.Common.csproj
@@ -23,6 +23,7 @@
+