From 962c9d1c953d7dbe3c7ef3897bafd94e3f9960e6 Mon Sep 17 00:00:00 2001 From: Bogdan Date: Mon, 14 Oct 2024 16:13:46 +0300 Subject: [PATCH] httpclient: treat "refresh" as redirect (#15646) --- .../Indexers/Definitions/CardigannIndexer.cs | 15 ++++++++-- .../Utils/Clients/HttpWebClient.cs | 8 +++-- .../Utils/Clients/HttpWebClient2.cs | 8 +++-- src/Jackett.Common/Utils/Clients/WebClient.cs | 29 +++++++++++++++++++ src/Jackett.Common/Utils/Clients/WebResult.cs | 3 +- 5 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/Jackett.Common/Indexers/Definitions/CardigannIndexer.cs b/src/Jackett.Common/Indexers/Definitions/CardigannIndexer.cs index 93ede8838..8b15ce5d9 100644 --- a/src/Jackett.Common/Indexers/Definitions/CardigannIndexer.cs +++ b/src/Jackett.Common/Indexers/Definitions/CardigannIndexer.cs @@ -867,17 +867,21 @@ namespace Jackett.Common.Indexers.Definitions var Login = Definition.Login; if (Login == null || Login.Test == null) + { return false; + } // test if login was successful - var LoginTestUrl = resolvePath(Login.Test.Path).ToString(); + var loginTestUrl = resolvePath(Login.Test.Path).ToString(); var headers = ParseCustomHeaders(Definition.Login?.Headers ?? Definition.Search?.Headers, GetBaseTemplateVariables()); - var testResult = await RequestWithCookiesAsync(LoginTestUrl, headers: headers); + var testResult = await RequestWithCookiesAsync(loginTestUrl, headers: headers); // Follow the redirect on login if the domain doesn't change if (testResult.IsRedirect && GetRedirectDomainHint(testResult) == null) { - testResult = await FollowIfRedirect(testResult, LoginTestUrl, overrideCookies: testResult.Cookies, accumulateCookies: true, maxRedirects: 1); + logger.Warn("Redirected to {0} from test login request", testResult.RedirectingTo); + + testResult = await FollowIfRedirect(testResult, loginTestUrl, overrideCookies: testResult.Cookies, accumulateCookies: true, maxRedirects: 1); } if (testResult.IsRedirect) @@ -888,6 +892,7 @@ namespace Jackett.Common.Indexers.Definitions if (domainHint != null) { errormessage += " Try changing the indexer URL to " + domainHint + "."; + if (Definition.Followredirect) { configData.SiteLink.Value = domainHint; @@ -896,6 +901,7 @@ namespace Jackett.Common.Indexers.Definitions errormessage += " Updated site link, please try again."; } } + throw new ExceptionWithConfigData(errormessage, configData); } @@ -903,12 +909,15 @@ namespace Jackett.Common.Indexers.Definitions { var testResultParser = new HtmlParser(); using var testResultDocument = testResultParser.ParseDocument(testResult.ContentString); + var selection = testResultDocument.QuerySelectorAll(Login.Test.Selector); + if (selection.Length == 0) { throw new ExceptionWithConfigData(string.Format("Login failed: Selector \"{0}\" didn't match", Login.Test.Selector), configData); } } + return true; } diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient.cs index a4806b545..490a05a4a 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient.cs @@ -194,10 +194,14 @@ namespace Jackett.Common.Utils.Clients } } } - if (response.Headers.Location != null) + + var redirectUri = RedirectUri(response); + + if (redirectUri != null) { - result.RedirectingTo = response.Headers.Location.ToString(); + result.RedirectingTo = redirectUri.AbsoluteUri; } + // Mono won't add the baseurl to relative redirects. // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php" // See issue #1200 diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs index fbd27f547..7fe2b468a 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs @@ -217,10 +217,14 @@ namespace Jackett.Common.Utils.Clients } } } - if (responseMessage.Headers.Location != null) + + var redirectUri = RedirectUri(responseMessage); + + if (redirectUri != null) { - result.RedirectingTo = responseMessage.Headers.Location.ToString(); + result.RedirectingTo = redirectUri.AbsoluteUri; } + // Mono won't add the baseurl to relative redirects. // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php" // See issue #1200 diff --git a/src/Jackett.Common/Utils/Clients/WebClient.cs b/src/Jackett.Common/Utils/Clients/WebClient.cs index 1a1cdbdf7..2b10c61d3 100644 --- a/src/Jackett.Common/Utils/Clients/WebClient.cs +++ b/src/Jackett.Common/Utils/Clients/WebClient.cs @@ -5,6 +5,7 @@ using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; +using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Web; using com.LandonKey.SocksWebProxy; @@ -17,6 +18,8 @@ namespace Jackett.Common.Utils.Clients { public abstract class WebClient : IObserver { + private static readonly Regex _RefreshHeaderRegex = new Regex("^(.*?url)=(.*?)(?:;|$)", RegexOptions.Compiled); + protected IDisposable ServerConfigUnsubscriber; protected Logger logger; protected IConfigurationService configService; @@ -263,5 +266,31 @@ namespace Jackett.Common.Utils.Clients content.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded"); return content; } + + protected static Uri RedirectUri(HttpResponseMessage response) + { + var newUri = response.Headers.Location; + + if (newUri == null && response.Headers.TryGetValues("Refresh", out var refreshHeaders)) + { + var refreshHeader = refreshHeaders.FirstOrDefault(); + + if (refreshHeader == null) + { + return null; + } + + var match = _RefreshHeaderRegex.Match(refreshHeader); + + if (match.Success) + { + return new Uri(response.RequestMessage.RequestUri, match.Groups[2].Value); + } + + return null; + } + + return newUri; + } } } diff --git a/src/Jackett.Common/Utils/Clients/WebResult.cs b/src/Jackett.Common/Utils/Clients/WebResult.cs index a11ca1ae7..f575df786 100644 --- a/src/Jackett.Common/Utils/Clients/WebResult.cs +++ b/src/Jackett.Common/Utils/Clients/WebResult.cs @@ -74,6 +74,7 @@ namespace Jackett.Common.Utils.Clients Status == HttpStatusCode.RedirectKeepVerb || Status == HttpStatusCode.RedirectMethod || Status == HttpStatusCode.Found || - Status == HttpStatusCode.MovedPermanently; + Status == HttpStatusCode.MovedPermanently || + Headers.ContainsKey("Refresh"); } }