From 7eaff55955a88d4a95a6d4d09e28d3d13cd4cf80 Mon Sep 17 00:00:00 2001 From: kaso17 Date: Fri, 7 Oct 2016 08:50:15 +0200 Subject: [PATCH] Support CloudFlare challenges with mono/libcurl (#538) * Add CloudFlare support for the libcurl WebClient * Save config if cookies are updated If the cookieheader/config isn't updated with e.g. the cf_clearance cookie jackett has to recompute the challenge on every request. --- src/Jackett/Indexers/BaseIndexer.cs | 23 ++++++- .../Utils/Clients/UnixLibCurlWebClient.cs | 60 +++++++++++++++++-- 2 files changed, 76 insertions(+), 7 deletions(-) diff --git a/src/Jackett/Indexers/BaseIndexer.cs b/src/Jackett/Indexers/BaseIndexer.cs index 481fdb709..af4391381 100644 --- a/src/Jackett/Indexers/BaseIndexer.cs +++ b/src/Jackett/Indexers/BaseIndexer.cs @@ -179,7 +179,7 @@ namespace Jackett.Indexers private String ResolveCookies(String incomingCookies = "") { - var redirRequestCookies = (CookieHeader != "" ? CookieHeader + " " : "") + incomingCookies; + var redirRequestCookies = (CookieHeader != null && CookieHeader != "" ? CookieHeader + " " : "") + incomingCookies; System.Text.RegularExpressions.Regex expression = new System.Text.RegularExpressions.Regex(@"([^\s]+)=([^=]+)(?:\s|$)"); Dictionary cookieDIctionary = new Dictionary(); var matches = expression.Match(redirRequestCookies); @@ -192,6 +192,19 @@ namespace Jackett.Indexers } + // Update CookieHeader with new cookies and save the config if something changed (e.g. a new CloudFlare clearance cookie was issued) + protected void UpdateCookieHeader(string newCookies, string cookieOverride = null) + { + string newCookieHeader = ResolveCookies((cookieOverride != null && cookieOverride != "" ? cookieOverride + " " : "") + newCookies); + if (CookieHeader != newCookieHeader) + { + logger.Debug(string.Format("updating Cookies {0} => {1}", CookieHeader, newCookieHeader)); + CookieHeader = newCookieHeader; + if (IsConfigured) + SaveConfig(); + } + } + private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false) { if (incomingResponse.IsRedirect) @@ -306,7 +319,9 @@ namespace Jackett.Indexers if (cookieOverride != null) request.Cookies = cookieOverride; - return await webclient.GetString(request); + WebClientStringResult result = await webclient.GetString(request); + UpdateCookieHeader(result.Cookies, cookieOverride); + return result; } protected async Task RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null, Dictionary headers = null) @@ -358,7 +373,9 @@ namespace Jackett.Indexers if (emulateBrowser.HasValue) request.EmulateBrowser = emulateBrowser.Value; - return await webclient.GetString(request); + WebClientStringResult result = await webclient.GetString(request); + UpdateCookieHeader(result.Cookies, cookieOverride); + return result; } protected async Task PostDataWithCookiesAndRetry(string url, IEnumerable> data, string cookieOverride = null, string referer = null, Dictionary headers = null, string rawbody = null, bool? emulateBrowser = null) diff --git a/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs b/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs index 1bb92ee0d..4e5d35c0b 100644 --- a/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs +++ b/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs @@ -11,22 +11,31 @@ using System.Net; using System.Net.Http; using System.Text; using System.Threading.Tasks; - +using System.Reflection; + namespace Jackett.Utils.Clients { public class UnixLibCurlWebClient : IWebClient { private Logger logger; + private Assembly CloudFlareUtilities; + private MethodInfo ChallengeSolverSolveMethod; + private MethodInfo ChallengeSolutionGetClearanceQueryMethod; public UnixLibCurlWebClient(Logger l) { logger = l; + + // use reflections to get the internal CloudFlareUtilities methods we need + CloudFlareUtilities = Assembly.LoadFile(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/CloudFlareUtilities.dll"); + ChallengeSolverSolveMethod = CloudFlareUtilities.GetType("CloudFlareUtilities.ChallengeSolver").GetMethod("Solve"); + ChallengeSolutionGetClearanceQueryMethod = CloudFlareUtilities.GetType("CloudFlareUtilities.ChallengeSolution").GetMethod("get_ClearanceQuery"); } public async Task GetBytes(WebRequest request) { logger.Debug(string.Format("UnixLibCurlWebClient:GetBytes(Url:{0})", request.Url)); - var result = await Run(request); + var result = await RunCloudFlare(request); logger.Debug(string.Format("UnixLibCurlWebClient:GetBytes Returning {0} => {1} bytes", result.Status, (result.Content == null ? "" : result.Content.Length.ToString()))); return result; } @@ -34,11 +43,19 @@ namespace Jackett.Utils.Clients public async Task GetString(WebRequest request) { logger.Debug(string.Format("UnixLibCurlWebClient:GetString(Url:{0})", request.Url)); - var result = await Run(request); + var result = await RunCloudFlare(request); logger.Debug(string.Format("UnixLibCurlWebClient:GetString Returning {0} => {1}", result.Status, (result.Content == null ? "" : Encoding.UTF8.GetString(result.Content)))); return Mapper.Map(result); } + private string CloudFlareChallengeSolverSolve(string challengePageContent, Uri uri) + { + var solution = ChallengeSolverSolveMethod.Invoke(null, new object[] { challengePageContent, uri.Host }); + string clearanceQuery = (string)ChallengeSolutionGetClearanceQueryMethod.Invoke(solution, new object[] { }); + string clearanceUri = uri.Scheme + Uri.SchemeDelimiter + uri.Host + ":" + uri.Port + clearanceQuery; + return clearanceUri; + } + public void Init() { try @@ -67,6 +84,41 @@ namespace Jackett.Utils.Clients } } + // Wrapper for Run which takes care of CloudFlare challenges + private async Task RunCloudFlare(WebRequest request) + { + WebClientByteResult result = await Run(request); + + // check if we've received a CloudFlare challenge + if (result.Status == HttpStatusCode.ServiceUnavailable && ((request.Cookies != null && request.Cookies.Contains("__cfduid")) || result.Cookies.Contains("__cfduid"))) + { + logger.Info("UnixLibCurlWebClient: Received a new CloudFlare challenge"); + + // solve the challenge + string pageContent = Encoding.UTF8.GetString(result.Content); + Uri uri = new Uri(request.Url); + string clearanceUri = CloudFlareChallengeSolverSolve(pageContent, uri); + logger.Info(string.Format("UnixLibCurlWebClient: CloudFlare clearanceUri: {0}", clearanceUri)); + + // wait... + await Task.Delay(5000); + + // request clearanceUri to get cf_clearance cookie + var response = await CurlHelper.GetAsync(clearanceUri, request.Cookies, request.Referer); + logger.Info(string.Format("UnixLibCurlWebClient: received CloudFlare clearance cookie: {0}", response.Cookies)); + + // add new cf_clearance cookies to the original request + request.Cookies = response.Cookies + request.Cookies; + + // re-run the original request with updated cf_clearance cookie + result = await Run(request); + + // add cf_clearance cookie to the final result so we update the config for the next request + result.Cookies = response.Cookies + " " + result.Cookies; + } + return result; + } + private async Task Run(WebRequest request) { Jackett.CurlHelper.CurlResponse response; @@ -85,7 +137,7 @@ namespace Jackett.Utils.Clients logger.Debug("UnixLibCurlWebClient: Posting " + StringUtil.PostDataFromDict(request.PostData)); } - response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.RawBody); + response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.RawBody); } var result = new WebClientByteResult()