shpaq/Jackett-Jackett
1
0
Fork 0
mirror of https://github.com/Jackett/Jackett.git synced 2025-09-17 17:34:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Support CloudFlare challenges with mono/libcurl (#538)

Browse Source 
* Add CloudFlare support for the libcurl WebClient

* Save config if cookies are updated

If the cookieheader/config isn't updated with e.g. the cf_clearance cookie jackett has to recompute the challenge on every request.
This commit is contained in:
kaso17
2016-10-07 08:50:15 +02:00
committed by JigSaw
parent 71414b805b
commit 7eaff55955
2 changed files with 76 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/Jackett/Indexers/BaseIndexer.cs
View File

@@ -179,7 +179,7 @@ namespace Jackett.Indexers
private String ResolveCookies(String incomingCookies = "")
{
var redirRequestCookies = (CookieHeader != "" ? CookieHeader + " " : "") + incomingCookies;
var redirRequestCookies = (CookieHeader != null && CookieHeader != "" ? CookieHeader + " " : "") + incomingCookies;
System.Text.RegularExpressions.Regex expression = new System.Text.RegularExpressions.Regex(@"([^\s]+)=([^=]+)(?:\s|$)");
Dictionary<string, string> cookieDIctionary = new Dictionary<string, string>();
var matches = expression.Match(redirRequestCookies);
@@ -192,6 +192,19 @@ namespace Jackett.Indexers
}
// Update CookieHeader with new cookies and save the config if something changed (e.g. a new CloudFlare clearance cookie was issued)
protected void UpdateCookieHeader(string newCookies, string cookieOverride = null)
{
string newCookieHeader = ResolveCookies((cookieOverride != null && cookieOverride != "" ? cookieOverride + " " : "") + newCookies);
if (CookieHeader != newCookieHeader)
{
logger.Debug(string.Format("updating Cookies {0} => {1}", CookieHeader, newCookieHeader));
CookieHeader = newCookieHeader;
if (IsConfigured)
SaveConfig();
}
}
private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false)
{
if (incomingResponse.IsRedirect)
@@ -306,7 +319,9 @@ namespace Jackett.Indexers
if (cookieOverride != null)
request.Cookies = cookieOverride;
return await webclient.GetString(request);
WebClientStringResult result = await webclient.GetString(request);
UpdateCookieHeader(result.Cookies, cookieOverride);
return result;
}
protected async Task<WebClientStringResult> RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null)
@@ -358,7 +373,9 @@ namespace Jackett.Indexers
if (emulateBrowser.HasValue)
request.EmulateBrowser = emulateBrowser.Value;
return await webclient.GetString(request);
WebClientStringResult result = await webclient.GetString(request);
UpdateCookieHeader(result.Cookies, cookieOverride);
return result;
}
protected async Task<WebClientStringResult> PostDataWithCookiesAndRetry(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)

60
src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs
View File

@@ -11,22 +11,31 @@ using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using System.Reflection;
namespace Jackett.Utils.Clients
{
public class UnixLibCurlWebClient : IWebClient
{
private Logger logger;
private Assembly CloudFlareUtilities;
private MethodInfo ChallengeSolverSolveMethod;
private MethodInfo ChallengeSolutionGetClearanceQueryMethod;
public UnixLibCurlWebClient(Logger l)
{
logger = l;
// use reflections to get the internal CloudFlareUtilities methods we need
CloudFlareUtilities = Assembly.LoadFile(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/CloudFlareUtilities.dll");
ChallengeSolverSolveMethod = CloudFlareUtilities.GetType("CloudFlareUtilities.ChallengeSolver").GetMethod("Solve");
ChallengeSolutionGetClearanceQueryMethod = CloudFlareUtilities.GetType("CloudFlareUtilities.ChallengeSolution").GetMethod("get_ClearanceQuery");
}
public async Task<WebClientByteResult> GetBytes(WebRequest request)
{
logger.Debug(string.Format("UnixLibCurlWebClient:GetBytes(Url:{0})", request.Url));
var result = await Run(request);
var result = await RunCloudFlare(request);
logger.Debug(string.Format("UnixLibCurlWebClient:GetBytes Returning {0} => {1} bytes", result.Status, (result.Content == null ? "<NULL>" : result.Content.Length.ToString())));
return result;
}
@@ -34,11 +43,19 @@ namespace Jackett.Utils.Clients
public async Task<WebClientStringResult> GetString(WebRequest request)
{
logger.Debug(string.Format("UnixLibCurlWebClient:GetString(Url:{0})", request.Url));
var result = await Run(request);
var result = await RunCloudFlare(request);
logger.Debug(string.Format("UnixLibCurlWebClient:GetString Returning {0} => {1}", result.Status, (result.Content == null ? "<NULL>" : Encoding.UTF8.GetString(result.Content))));
return Mapper.Map<WebClientStringResult>(result);
}
private string CloudFlareChallengeSolverSolve(string challengePageContent, Uri uri)
{
var solution = ChallengeSolverSolveMethod.Invoke(null, new object[] { challengePageContent, uri.Host });
string clearanceQuery = (string)ChallengeSolutionGetClearanceQueryMethod.Invoke(solution, new object[] { });
string clearanceUri = uri.Scheme + Uri.SchemeDelimiter + uri.Host + ":" + uri.Port + clearanceQuery;
return clearanceUri;
}
public void Init()
{
try
@@ -67,6 +84,41 @@ namespace Jackett.Utils.Clients
}
}
// Wrapper for Run which takes care of CloudFlare challenges
private async Task<WebClientByteResult> RunCloudFlare(WebRequest request)
{
WebClientByteResult result = await Run(request);
// check if we've received a CloudFlare challenge
if (result.Status == HttpStatusCode.ServiceUnavailable && ((request.Cookies != null && request.Cookies.Contains("__cfduid")) || result.Cookies.Contains("__cfduid")))
{
logger.Info("UnixLibCurlWebClient: Received a new CloudFlare challenge");
// solve the challenge
string pageContent = Encoding.UTF8.GetString(result.Content);
Uri uri = new Uri(request.Url);
string clearanceUri = CloudFlareChallengeSolverSolve(pageContent, uri);
logger.Info(string.Format("UnixLibCurlWebClient: CloudFlare clearanceUri: {0}", clearanceUri));
// wait...
await Task.Delay(5000);
// request clearanceUri to get cf_clearance cookie
var response = await CurlHelper.GetAsync(clearanceUri, request.Cookies, request.Referer);
logger.Info(string.Format("UnixLibCurlWebClient: received CloudFlare clearance cookie: {0}", response.Cookies));
// add new cf_clearance cookies to the original request
request.Cookies = response.Cookies + request.Cookies;
// re-run the original request with updated cf_clearance cookie
result = await Run(request);
// add cf_clearance cookie to the final result so we update the config for the next request
result.Cookies = response.Cookies + " " + result.Cookies;
}
return result;
}
private async Task<WebClientByteResult> Run(WebRequest request)
{
Jackett.CurlHelper.CurlResponse response;
@@ -85,7 +137,7 @@ namespace Jackett.Utils.Clients
logger.Debug("UnixLibCurlWebClient: Posting " + StringUtil.PostDataFromDict(request.PostData));
}
response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.RawBody);
response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.RawBody);
}
var result = new WebClientByteResult()