shpaq/Jackett-Jackett
1
0
Fork 0
mirror of https://github.com/Jackett/Jackett.git synced 2025-09-17 17:34:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

core: refactor http webclient part 13 #8529 (#8941)

Browse Source 
Merge string/byte request code
This commit is contained in:
Cory
2020-06-11 10:09:27 -05:00
committed by ngosang
parent 22bced9ca3
commit 4d63fa81d5
91 changed files with 330 additions and 399 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
src/Jackett.Common/Indexers/BaseIndexer.cs
View File

@@ -392,11 +392,11 @@ namespace Jackett.Common.Indexers
.Replace("(", "%28")
.Replace(")", "%29")
.Replace("'", "%27");
var response = await RequestBytesWithCookiesAndRetry(requestLink, null, method, requestLink);
var response = await RequestWithCookiesAndRetryAsync(requestLink, null, method, requestLink);
// if referer link is provied it will be used
if (refererlink != null)
response = await RequestBytesWithCookiesAndRetry(requestLink, null, method, refererlink);
response = await RequestWithCookiesAndRetryAsync(requestLink, null, method, refererlink);
if (response.IsRedirect)
{
await FollowIfRedirect(response);
@@ -412,69 +412,38 @@ namespace Jackett.Common.Indexers
return response.ContentBytes;
}
protected async Task<WebResult> RequestBytesWithCookiesAndRetry(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null)
protected async Task<WebResult> RequestWithCookiesAndRetryAsync(
string url, string cookieOverride = null, RequestType method = RequestType.GET,
string referer = null, IEnumerable<KeyValuePair<string, string>> data = null,
Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
{
Exception lastException = null;
for (var i = 0; i < 3; i++)
{
try
{
return await RequestBytesWithCookies(url, cookieOverride, method, referer, data);
return await WebRequestWithCookiesAsync(
url, cookieOverride, method, referer, data, headers, rawbody, emulateBrowser);
}
catch (Exception e)
{
logger.Error(e, string.Format("On attempt {0} downloading from {1}: {2}", (i + 1), DisplayName, e.Message));
logger.Error(
e, string.Format("On attempt {0} downloading from {1}: {2}", (i + 1), DisplayName, e.Message));
lastException = e;
}
await Task.Delay(500);
}
throw lastException;
}
protected async Task<WebResult> RequestStringWithCookies(string url, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null)
protected virtual async Task<WebResult> WebRequestWithCookiesAsync(
string url, string cookieOverride = null, RequestType method = RequestType.GET,
string referer = null, IEnumerable<KeyValuePair<string, string>> data = null,
Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
{
var request = new Utils.Clients.WebRequest()
{
Url = url,
Type = RequestType.GET,
Cookies = CookieHeader,
Referer = referer,
Headers = headers,
Encoding = Encoding
};
if (cookieOverride != null)
request.Cookies = cookieOverride;
var result = await webclient.GetString(request);
CheckTrackerDown(result);
UpdateCookieHeader(result.Cookies, cookieOverride);
return result;
}
protected async Task<WebResult> RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null)
{
Exception lastException = null;
for (var i = 0; i < 3; i++)
{
try
{
return await RequestStringWithCookies(url, cookieOverride, referer, headers);
}
catch (Exception e)
{
logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message));
lastException = e;
}
await Task.Delay(500);
}
throw lastException;
}
protected virtual async Task<WebResult> RequestBytesWithCookies(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable<KeyValuePair<string, string>> data = null, Dictionary<string, string> headers = null)
{
var request = new Utils.Clients.WebRequest()
var request = new WebRequest
{
Url = url,
Type = method,
@@ -482,58 +451,18 @@ namespace Jackett.Common.Indexers
PostData = data,
Referer = referer,
Headers = headers,
Encoding = Encoding
};
if (cookieOverride != null)
request.Cookies = cookieOverride;
var result = await webclient.GetBytes(request);
UpdateCookieHeader(result.Cookies, cookieOverride);
return result;
}
protected async Task<WebResult> PostDataWithCookies(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
{
var request = new Utils.Clients.WebRequest()
{
Url = url,
Type = RequestType.POST,
Cookies = cookieOverride ?? CookieHeader,
PostData = data,
Referer = referer,
Headers = headers,
RawBody = rawbody,
Encoding = Encoding
};
if (emulateBrowser.HasValue)
request.EmulateBrowser = emulateBrowser.Value;
var result = await webclient.GetString(request);
CheckTrackerDown(result);
var result = await webclient.GetResultAsync(request);
CheckSiteDown(result);
UpdateCookieHeader(result.Cookies, cookieOverride);
return result;
}
protected async Task<WebResult> PostDataWithCookiesAndRetry(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
{
Exception lastException = null;
for (var i = 0; i < 3; i++)
{
try
{
return await PostDataWithCookies(url, data, cookieOverride, referer, headers, rawbody, emulateBrowser);
}
catch (Exception e)
{
logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message));
lastException = e;
}
await Task.Delay(500);
}
throw lastException;
}
protected async Task<WebResult> RequestLoginAndFollowRedirect(string url, IEnumerable<KeyValuePair<string, string>> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null, bool accumulateCookies = false)
{
var request = new Utils.Clients.WebRequest()
@@ -545,8 +474,8 @@ namespace Jackett.Common.Indexers
PostData = data,
Encoding = Encoding
};
var response = await webclient.GetString(request);
CheckTrackerDown(response);
var response = await webclient.GetResultAsync(request);
CheckSiteDown(response);
if (accumulateCookies)
{
response.Cookies = ResolveCookies((request.Cookies == null ? "" : request.Cookies + " ") + response.Cookies);
@@ -566,7 +495,7 @@ namespace Jackett.Common.Indexers
return response;
}
protected void CheckTrackerDown(WebResult response)
protected static void CheckSiteDown(WebResult response)
{
if (response.Status == System.Net.HttpStatusCode.BadGateway
|| response.Status == System.Net.HttpStatusCode.GatewayTimeout
@@ -643,7 +572,7 @@ namespace Jackett.Common.Indexers
redirRequestCookies = (overrideCookies != null ? overrideCookies : "");
}
// Do redirect
var redirectedResponse = await webclient.GetBytes(new WebRequest()
var redirectedResponse = await webclient.GetResultAsync(new WebRequest()
{
Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo,
Referer = referrer,