httpclient: treat "refresh" as redirect (#15646)

src/Jackett.Common/Indexers/Definitions/CardigannIndexer.cs

@@ -867,17 +867,21 @@ namespace Jackett.Common.Indexers.Definitions
             var Login = Definition.Login;
 
             if (Login == null || Login.Test == null)
+            {
                 return false;
+            }
 
             // test if login was successful
-            var LoginTestUrl = resolvePath(Login.Test.Path).ToString();
+            var loginTestUrl = resolvePath(Login.Test.Path).ToString();
             var headers = ParseCustomHeaders(Definition.Login?.Headers ?? Definition.Search?.Headers, GetBaseTemplateVariables());
-            var testResult = await RequestWithCookiesAsync(LoginTestUrl, headers: headers);
+            var testResult = await RequestWithCookiesAsync(loginTestUrl, headers: headers);
 
             // Follow the redirect on login if the domain doesn't change
             if (testResult.IsRedirect && GetRedirectDomainHint(testResult) == null)
             {
-                testResult = await FollowIfRedirect(testResult, LoginTestUrl, overrideCookies: testResult.Cookies, accumulateCookies: true, maxRedirects: 1);
+                logger.Warn("Redirected to {0} from test login request", testResult.RedirectingTo);
+
+                testResult = await FollowIfRedirect(testResult, loginTestUrl, overrideCookies: testResult.Cookies, accumulateCookies: true, maxRedirects: 1);
             }
 
             if (testResult.IsRedirect)
@@ -888,6 +892,7 @@ namespace Jackett.Common.Indexers.Definitions
                 if (domainHint != null)
                 {
                     errormessage += " Try changing the indexer URL to " + domainHint + ".";
+
                     if (Definition.Followredirect)
                     {
                         configData.SiteLink.Value = domainHint;
@@ -896,6 +901,7 @@ namespace Jackett.Common.Indexers.Definitions
                         errormessage += " Updated site link, please try again.";
                     }
                 }
+
                 throw new ExceptionWithConfigData(errormessage, configData);
             }
 
@@ -903,12 +909,15 @@ namespace Jackett.Common.Indexers.Definitions
             {
                 var testResultParser = new HtmlParser();
                 using var testResultDocument = testResultParser.ParseDocument(testResult.ContentString);
+
                 var selection = testResultDocument.QuerySelectorAll(Login.Test.Selector);
+
                 if (selection.Length == 0)
                 {
                     throw new ExceptionWithConfigData(string.Format("Login failed: Selector \"{0}\" didn't match", Login.Test.Selector), configData);
                 }
             }
+
             return true;
         }
 

src/Jackett.Common/Utils/Clients/HttpWebClient.cs

@@ -194,10 +194,14 @@ namespace Jackett.Common.Utils.Clients
                                         }
                                     }
                                 }
-                                if (response.Headers.Location != null)
+
+                                var redirectUri = RedirectUri(response);
+
+                                if (redirectUri != null)
                                 {
-                                    result.RedirectingTo = response.Headers.Location.ToString();
+                                    result.RedirectingTo = redirectUri.AbsoluteUri;
                                 }
+
                                 // Mono won't add the baseurl to relative redirects.
                                 // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
                                 // See issue #1200

src/Jackett.Common/Utils/Clients/HttpWebClient2.cs

@@ -217,10 +217,14 @@ namespace Jackett.Common.Utils.Clients
                     }
                 }
             }
-            if (responseMessage.Headers.Location != null)
+
+            var redirectUri = RedirectUri(responseMessage);
+
+            if (redirectUri != null)
             {
-                result.RedirectingTo = responseMessage.Headers.Location.ToString();
+                result.RedirectingTo = redirectUri.AbsoluteUri;
             }
+
             // Mono won't add the baseurl to relative redirects.
             // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
             // See issue #1200

src/Jackett.Common/Utils/Clients/WebClient.cs

@@ -5,6 +5,7 @@ using System.Net;
 using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using System.Web;
 using com.LandonKey.SocksWebProxy;
@@ -17,6 +18,8 @@ namespace Jackett.Common.Utils.Clients
 {
     public abstract class WebClient : IObserver<ServerConfig>
     {
+        private static readonly Regex _RefreshHeaderRegex = new Regex("^(.*?url)=(.*?)(?:;|$)", RegexOptions.Compiled);
+
         protected IDisposable ServerConfigUnsubscriber;
         protected Logger logger;
         protected IConfigurationService configService;
@@ -263,5 +266,31 @@ namespace Jackett.Common.Utils.Clients
             content.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded");
             return content;
         }
+
+        protected static Uri RedirectUri(HttpResponseMessage response)
+        {
+            var newUri = response.Headers.Location;
+
+            if (newUri == null && response.Headers.TryGetValues("Refresh", out var refreshHeaders))
+            {
+                var refreshHeader = refreshHeaders.FirstOrDefault();
+
+                if (refreshHeader == null)
+                {
+                    return null;
+                }
+
+                var match = _RefreshHeaderRegex.Match(refreshHeader);
+
+                if (match.Success)
+                {
+                    return new Uri(response.RequestMessage.RequestUri, match.Groups[2].Value);
+                }
+
+                return null;
+            }
+
+            return newUri;
+        }
     }
 }

src/Jackett.Common/Utils/Clients/WebResult.cs

@@ -74,6 +74,7 @@ namespace Jackett.Common.Utils.Clients
                                   Status == HttpStatusCode.RedirectKeepVerb ||
                                   Status == HttpStatusCode.RedirectMethod ||
                                   Status == HttpStatusCode.Found ||
-                                  Status == HttpStatusCode.MovedPermanently;
+                                  Status == HttpStatusCode.MovedPermanently ||
+                                  Headers.ContainsKey("Refresh");
     }
 }