diff --git a/src/Jackett.Common/Indexers/Abnormal.cs b/src/Jackett.Common/Indexers/Abnormal.cs index 941a5a889..932c7715d 100644 --- a/src/Jackett.Common/Indexers/Abnormal.cs +++ b/src/Jackett.Common/Indexers/Abnormal.cs @@ -21,6 +21,7 @@ using Jackett.Common.Utils.Clients; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; +using WebRequest = Jackett.Common.Utils.Clients.WebRequest; namespace Jackett.Common.Indexers { @@ -166,7 +167,7 @@ namespace Jackett.Common.Indexers // Perform loggin latencyNow(); output("\nPerform loggin.. with " + LoginUrl); - var response = await webclient.GetString(request); + var response = await webclient.GetResultAsync(request); // Test if we are logged in await ConfigureIfOK(response.Cookies, response.Cookies.Contains("session="), () => @@ -548,14 +549,12 @@ namespace Jackett.Common.Indexers /// Results from query private async Task queryTracker(string request) { - WebResult results = null; - // Cache mode not enabled or cached file didn't exist for our query output("\nQuerying tracker for results...."); // Request our first page latencyNow(); - results = await RequestStringWithCookiesAndRetry(request, null, null, emulatedBrowserHeaders); + var results = await RequestWithCookiesAndRetryAsync(request, headers: emulatedBrowserHeaders); // Return results from tracker return results.ContentString; diff --git a/src/Jackett.Common/Indexers/Abstract/AvistazTracker.cs b/src/Jackett.Common/Indexers/Abstract/AvistazTracker.cs index 20f4b66af..4caaab8a2 100644 --- a/src/Jackett.Common/Indexers/Abstract/AvistazTracker.cs +++ b/src/Jackett.Common/Indexers/Abstract/AvistazTracker.cs @@ -10,6 +10,7 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using WebClient = Jackett.Common.Utils.Clients.WebClient; @@ -148,7 +149,7 @@ without this configuration the torrent download does not work.
You can find { "password", configData.Password.Value.Trim() }, { "pid", configData.Pid.Value.Trim() } }; - var result = await PostDataWithCookies(AuthUrl, body, headers: AuthHeaders); + var result = await WebRequestWithCookiesAsync(AuthUrl, method: RequestType.POST, data: body, headers: AuthHeaders); var json = JObject.Parse(result.ContentString); _token = json.Value("token"); if (_token == null) @@ -161,11 +162,11 @@ without this configuration the torrent download does not work.
You can find var qc = GetSearchQueryParameters(query); var episodeSearchUrl = SearchUrl + "?" + qc.GetQueryString(); - var response = await RequestStringWithCookiesAndRetry(episodeSearchUrl, headers: GetSearchHeaders()); + var response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl, headers: GetSearchHeaders()); if (response.Status == HttpStatusCode.Unauthorized || response.Status == HttpStatusCode.PreconditionFailed) { await RenewalTokenAsync(); - response = await RequestStringWithCookiesAndRetry(episodeSearchUrl, headers: GetSearchHeaders()); + response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl, headers: GetSearchHeaders()); } else if (response.Status != HttpStatusCode.OK) throw new Exception($"Unknown error: {response.ContentString}"); diff --git a/src/Jackett.Common/Indexers/Abstract/CouchPotatoTracker.cs b/src/Jackett.Common/Indexers/Abstract/CouchPotatoTracker.cs index fdba46f78..feeb815a4 100644 --- a/src/Jackett.Common/Indexers/Abstract/CouchPotatoTracker.cs +++ b/src/Jackett.Common/Indexers/Abstract/CouchPotatoTracker.cs @@ -75,7 +75,7 @@ namespace Jackett.Common.Indexers.Abstract searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestStringWithCookiesAndRetry(searchUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl); JObject json = null; try diff --git a/src/Jackett.Common/Indexers/Abstract/GazelleTracker.cs b/src/Jackett.Common/Indexers/Abstract/GazelleTracker.cs index ececf7356..f0b245827 100644 --- a/src/Jackett.Common/Indexers/Abstract/GazelleTracker.cs +++ b/src/Jackett.Common/Indexers/Abstract/GazelleTracker.cs @@ -12,6 +12,7 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using WebClient = Jackett.Common.Utils.Clients.WebClient; @@ -196,12 +197,12 @@ namespace Jackett.Common.Indexers.Abstract searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestStringWithCookiesAndRetry(searchUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl); if (response.IsRedirect) { // re-login await ApplyConfiguration(null); - response = await RequestStringWithCookiesAndRetry(searchUrl); + response = await RequestWithCookiesAndRetryAsync(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/Abstract/XtremeZoneTracker.cs b/src/Jackett.Common/Indexers/Abstract/XtremeZoneTracker.cs index ddeb717ba..7d7aa779c 100644 --- a/src/Jackett.Common/Indexers/Abstract/XtremeZoneTracker.cs +++ b/src/Jackett.Common/Indexers/Abstract/XtremeZoneTracker.cs @@ -10,6 +10,7 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; @@ -74,7 +75,8 @@ namespace Jackett.Common.Indexers.Abstract { "password", configData.Password.Value.Trim() } }; var jsonData = JsonConvert.SerializeObject(body); - var result = await PostDataWithCookies(LoginUrl, null, headers: ApiHeaders, rawbody: jsonData); + var result = await WebRequestWithCookiesAsync( + LoginUrl, method: RequestType.POST, headers: ApiHeaders, rawbody: jsonData); var json = JObject.Parse(result.ContentString); _token = json.Value("token"); if (_token == null) @@ -105,11 +107,11 @@ namespace Jackett.Common.Indexers.Abstract await RenewalTokenAsync(); var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var response = await RequestStringWithCookies(searchUrl, headers: GetSearchHeaders()); + var response = await WebRequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders()); if (response.Status == HttpStatusCode.Unauthorized) { await RenewalTokenAsync(); // re-login - response = await RequestStringWithCookies(searchUrl, headers: GetSearchHeaders()); + response = await WebRequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders()); } else if (response.Status != HttpStatusCode.OK) throw new Exception($"Unknown error in search: {response.ContentString}"); @@ -167,11 +169,11 @@ namespace Jackett.Common.Indexers.Abstract public override async Task Download(Uri link) { - var response = await RequestBytesWithCookies(link.ToString(), headers: GetSearchHeaders()); + var response = await WebRequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders()); if (response.Status == HttpStatusCode.Unauthorized) { await RenewalTokenAsync(); - response = await RequestBytesWithCookies(link.ToString(), headers: GetSearchHeaders()); + response = await WebRequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders()); } else if (response.Status != HttpStatusCode.OK) throw new Exception($"Unknown error in download: {response.ContentBytes}"); diff --git a/src/Jackett.Common/Indexers/AniDUB.cs b/src/Jackett.Common/Indexers/AniDUB.cs index 15181a204..6f86f138f 100644 --- a/src/Jackett.Common/Indexers/AniDUB.cs +++ b/src/Jackett.Common/Indexers/AniDUB.cs @@ -145,7 +145,7 @@ namespace Jackett.Common.Indexers private async Task EnsureAuthorized() { - var result = await RequestStringWithCookiesAndRetry(SiteLink); + var result = await RequestWithCookiesAndRetryAsync(SiteLink); if (!IsAuthorized(result)) { @@ -156,8 +156,7 @@ namespace Jackett.Common.Indexers private async Task> FetchNewReleases() { const string ReleaseLinksSelector = "#dle-content > .story > .story_h > .lcol > h2 > a"; - - var result = await RequestStringWithCookiesAndRetry(SiteLink); + var result = await RequestWithCookiesAndRetryAsync(SiteLink); var releases = new List(); try @@ -195,7 +194,7 @@ namespace Jackett.Common.Indexers return releases; } - var result = await RequestStringWithCookiesAndRetry(url); + var result = await RequestWithCookiesAndRetryAsync(url); try { @@ -529,8 +528,7 @@ namespace Jackett.Common.Indexers const string searchLinkSelector = "#dle-content > .searchitem > h3 > a"; var releases = new List(); - - var response = await PostDataWithCookiesAndRetry(SearchUrl, PreparePostData(query)); + var response = await RequestWithCookiesAndRetryAsync(SearchUrl, method: RequestType.POST, data: PreparePostData(query)); try { diff --git a/src/Jackett.Common/Indexers/Anidex.cs b/src/Jackett.Common/Indexers/Anidex.cs index c91f1a010..79b957eea 100644 --- a/src/Jackett.Common/Indexers/Anidex.cs +++ b/src/Jackett.Common/Indexers/Anidex.cs @@ -14,6 +14,7 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using static Jackett.Common.Models.IndexerConfig.ConfigurationData; @@ -150,13 +151,13 @@ namespace Jackett.Common.Indexers // Make search request var searchUri = GetAbsoluteUrl("?" + queryParameters.GetQueryString()); - var response = await RequestStringWithCookiesAndRetry(searchUri.AbsoluteUri); + var response = await RequestWithCookiesAndRetryAsync(searchUri.AbsoluteUri); // Check for DDOS Guard if (response.Status == System.Net.HttpStatusCode.Forbidden) { await ConfigureDDoSGuardCookie(); - response = await RequestStringWithCookiesAndRetry(searchUri.AbsoluteUri); + response = await RequestWithCookiesAndRetryAsync(searchUri.AbsoluteUri); } if (response.Status != System.Net.HttpStatusCode.OK) @@ -216,7 +217,7 @@ namespace Jackett.Common.Indexers private async Task ConfigureDDoSGuardCookie() { const string ddosPostUrl = "https://check.ddos-guard.net/check.js"; - var response = await RequestStringWithCookies(ddosPostUrl, string.Empty); + var response = await WebRequestWithCookiesAsync(ddosPostUrl, string.Empty); if (response.Status != System.Net.HttpStatusCode.OK) throw new WebException($"Unexpected DDOS Guard response: Status: {response.Status}", WebExceptionStatus.ProtocolError); if (response.IsRedirect) diff --git a/src/Jackett.Common/Indexers/AnimeBytes.cs b/src/Jackett.Common/Indexers/AnimeBytes.cs index 0859e81a5..4bc5c6548 100644 --- a/src/Jackett.Common/Indexers/AnimeBytes.cs +++ b/src/Jackett.Common/Indexers/AnimeBytes.cs @@ -12,6 +12,7 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig.Bespoke; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; @@ -179,7 +180,7 @@ namespace Jackett.Common.Indexers } // Get the content from the tracker - var response = await RequestStringWithCookiesAndRetry(queryUrl); + var response = await RequestWithCookiesAndRetryAsync(queryUrl); if (!response.ContentString.StartsWith("{")) // not JSON => error throw new ExceptionWithConfigData("unexcepted response (not JSON)", configData); dynamic json = JsonConvert.DeserializeObject(response.ContentString); diff --git a/src/Jackett.Common/Indexers/AnimeTorrents.cs b/src/Jackett.Common/Indexers/AnimeTorrents.cs index 97ad501e8..50b2b8ae3 100644 --- a/src/Jackett.Common/Indexers/AnimeTorrents.cs +++ b/src/Jackett.Common/Indexers/AnimeTorrents.cs @@ -1,4 +1,4 @@ -using System; + using System; using System.Collections.Generic; using System.Collections.Specialized; using System.Diagnostics.CodeAnalysis; @@ -79,7 +79,7 @@ namespace Jackett.Common.Indexers { "rememberme[]", "1" } }; - var loginPage = await RequestStringWithCookiesAndRetry(LoginUrl, "", LoginUrl); + var loginPage = await RequestWithCookiesAndRetryAsync(LoginUrl, "", RequestType.GET, LoginUrl); var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true); await ConfigureIfOK(result.Cookies, result.ContentString != null && result.ContentString.Contains("logout.php"), () => @@ -117,7 +117,8 @@ namespace Jackett.Common.Indexers { "X-Requested-With", "XMLHttpRequest" } }; - var response = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrlReferer, extraHeaders); + var response = await RequestWithCookiesAndRetryAsync( + searchUrl, referer: SearchUrlReferer, headers: extraHeaders); var results = response.ContentString; try diff --git a/src/Jackett.Common/Indexers/Anthelion.cs b/src/Jackett.Common/Indexers/Anthelion.cs index 2145010cb..2f2335afc 100644 --- a/src/Jackett.Common/Indexers/Anthelion.cs +++ b/src/Jackett.Common/Indexers/Anthelion.cs @@ -94,7 +94,7 @@ namespace Jackett.Common.Indexers qc.Add($"filter_cat[{cat}]", "1"); var searchUrl = BrowseUrl + "?" + qc.GetQueryString(); - var results = await RequestStringWithCookies(searchUrl); + var results = await WebRequestWithCookiesAsync(searchUrl); try { var parser = new HtmlParser(); diff --git a/src/Jackett.Common/Indexers/AwesomeHD.cs b/src/Jackett.Common/Indexers/AwesomeHD.cs index 8b06914d3..d91888d7d 100644 --- a/src/Jackett.Common/Indexers/AwesomeHD.cs +++ b/src/Jackett.Common/Indexers/AwesomeHD.cs @@ -100,7 +100,7 @@ namespace Jackett.Common.Indexers } var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var results = await RequestStringWithCookies(searchUrl); + var results = await WebRequestWithCookiesAsync(searchUrl); if (string.IsNullOrWhiteSpace(results.ContentString)) throw new Exception("Empty response. Please, check the Passkey."); diff --git a/src/Jackett.Common/Indexers/BB.cs b/src/Jackett.Common/Indexers/BB.cs index 36ed6f9fe..6614b2b52 100644 --- a/src/Jackett.Common/Indexers/BB.cs +++ b/src/Jackett.Common/Indexers/BB.cs @@ -122,8 +122,8 @@ namespace Jackett.Common.Indexers request_urls.Add(SearchUrl + queryCollection.GetQueryString()); } - var downloadTasksQuery = - from url in request_urls select RequestStringWithCookiesAndRetry(url); + + var downloadTasksQuery = from url in request_urls select RequestWithCookiesAndRetryAsync(url); var responses = await Task.WhenAll(downloadTasksQuery.ToArray()); @@ -134,7 +134,7 @@ namespace Jackett.Common.Indexers if (results.IsRedirect) { await ApplyConfiguration(null); - results = await RequestStringWithCookiesAndRetry(request_urls[i]); + results = await RequestWithCookiesAndRetryAsync(request_urls[i]); } try { diff --git a/src/Jackett.Common/Indexers/BJShare.cs b/src/Jackett.Common/Indexers/BJShare.cs index 9ea87c84b..e05405426 100644 --- a/src/Jackett.Common/Indexers/BJShare.cs +++ b/src/Jackett.Common/Indexers/BJShare.cs @@ -257,12 +257,12 @@ namespace Jackett.Common.Indexers foreach (var cat in MapTorznabCapsToTrackers(query)) queryCollection.Add("filter_cat[" + cat + "]", "1"); searchUrl += "?" + queryCollection.GetQueryString(); - var results = await RequestStringWithCookies(searchUrl); + var results = await WebRequestWithCookiesAsync(searchUrl); if (IsSessionIsClosed(results)) { // re-login await ApplyConfiguration(null); - results = await RequestStringWithCookies(searchUrl); + results = await WebRequestWithCookiesAsync(searchUrl); } try @@ -389,12 +389,12 @@ namespace Jackett.Common.Indexers private async Task> ParseLast24HoursAsync() { var releases = new List(); - var results = await RequestStringWithCookies(TodayUrl); + var results = await WebRequestWithCookiesAsync(TodayUrl); if (IsSessionIsClosed(results)) { // re-login await ApplyConfiguration(null); - results = await RequestStringWithCookies(TodayUrl); + results = await WebRequestWithCookiesAsync(TodayUrl); } try diff --git a/src/Jackett.Common/Indexers/BakaBT.cs b/src/Jackett.Common/Indexers/BakaBT.cs index 80efa758c..29f66a070 100644 --- a/src/Jackett.Common/Indexers/BakaBT.cs +++ b/src/Jackett.Common/Indexers/BakaBT.cs @@ -14,6 +14,7 @@ using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; +using WebRequest = Jackett.Common.Utils.Clients.WebRequest; namespace Jackett.Common.Indexers { @@ -68,7 +69,7 @@ namespace Jackett.Common.Indexers private async Task DoLogin() { - var loginForm = await webclient.GetString(new Utils.Clients.WebRequest() + var loginForm = await webclient.GetResultAsync(new Utils.Clients.WebRequest() { Url = LoginUrl, Type = RequestType.GET @@ -106,12 +107,12 @@ namespace Jackett.Common.Indexers var releases = new List(); var searchString = queryCopy.SanitizedSearchTerm; var episodeSearchUrl = SearchUrl + WebUtility.UrlEncode(searchString); - var response = await RequestStringWithCookiesAndRetry(episodeSearchUrl); + var response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl); if (!response.ContentString.Contains(LogoutStr)) { //Cookie appears to expire after a period of time or logging in to the site via browser await DoLogin(); - response = await RequestStringWithCookiesAndRetry(episodeSearchUrl); + response = await RequestWithCookiesAndRetryAsync(episodeSearchUrl); } try @@ -246,7 +247,7 @@ namespace Jackett.Common.Indexers public override async Task Download(Uri link) { - var downloadPage = await RequestStringWithCookies(link.ToString()); + var downloadPage = await WebRequestWithCookiesAsync(link.ToString()); var parser = new HtmlParser(); var dom = parser.ParseDocument(downloadPage.ContentString); var downloadLink = dom.QuerySelectorAll(".download_link").First().GetAttribute("href"); @@ -254,7 +255,7 @@ namespace Jackett.Common.Indexers if (string.IsNullOrWhiteSpace(downloadLink)) throw new Exception("Unable to find download link."); - var response = await RequestBytesWithCookies(SiteLink + downloadLink); + var response = await WebRequestWithCookiesAsync(SiteLink + downloadLink); return response.ContentBytes; } } diff --git a/src/Jackett.Common/Indexers/BaseIndexer.cs b/src/Jackett.Common/Indexers/BaseIndexer.cs index 5ab118370..859669c82 100644 --- a/src/Jackett.Common/Indexers/BaseIndexer.cs +++ b/src/Jackett.Common/Indexers/BaseIndexer.cs @@ -392,11 +392,11 @@ namespace Jackett.Common.Indexers .Replace("(", "%28") .Replace(")", "%29") .Replace("'", "%27"); - var response = await RequestBytesWithCookiesAndRetry(requestLink, null, method, requestLink); + var response = await RequestWithCookiesAndRetryAsync(requestLink, null, method, requestLink); // if referer link is provied it will be used if (refererlink != null) - response = await RequestBytesWithCookiesAndRetry(requestLink, null, method, refererlink); + response = await RequestWithCookiesAndRetryAsync(requestLink, null, method, refererlink); if (response.IsRedirect) { await FollowIfRedirect(response); @@ -412,69 +412,38 @@ namespace Jackett.Common.Indexers return response.ContentBytes; } - protected async Task RequestBytesWithCookiesAndRetry(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable> data = null) + protected async Task RequestWithCookiesAndRetryAsync( + string url, string cookieOverride = null, RequestType method = RequestType.GET, + string referer = null, IEnumerable> data = null, + Dictionary headers = null, string rawbody = null, bool? emulateBrowser = null) { Exception lastException = null; for (var i = 0; i < 3; i++) { try { - return await RequestBytesWithCookies(url, cookieOverride, method, referer, data); + return await WebRequestWithCookiesAsync( + url, cookieOverride, method, referer, data, headers, rawbody, emulateBrowser); } catch (Exception e) { - logger.Error(e, string.Format("On attempt {0} downloading from {1}: {2}", (i + 1), DisplayName, e.Message)); + logger.Error( + e, string.Format("On attempt {0} downloading from {1}: {2}", (i + 1), DisplayName, e.Message)); lastException = e; } + await Task.Delay(500); } throw lastException; } - protected async Task RequestStringWithCookies(string url, string cookieOverride = null, string referer = null, Dictionary headers = null) + protected virtual async Task WebRequestWithCookiesAsync( + string url, string cookieOverride = null, RequestType method = RequestType.GET, + string referer = null, IEnumerable> data = null, + Dictionary headers = null, string rawbody = null, bool? emulateBrowser = null) { - var request = new Utils.Clients.WebRequest() - { - Url = url, - Type = RequestType.GET, - Cookies = CookieHeader, - Referer = referer, - Headers = headers, - Encoding = Encoding - }; - - if (cookieOverride != null) - request.Cookies = cookieOverride; - var result = await webclient.GetString(request); - CheckTrackerDown(result); - UpdateCookieHeader(result.Cookies, cookieOverride); - return result; - } - - protected async Task RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null, Dictionary headers = null) - { - Exception lastException = null; - for (var i = 0; i < 3; i++) - { - try - { - return await RequestStringWithCookies(url, cookieOverride, referer, headers); - } - catch (Exception e) - { - logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message)); - lastException = e; - } - await Task.Delay(500); - } - - throw lastException; - } - - protected virtual async Task RequestBytesWithCookies(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable> data = null, Dictionary headers = null) - { - var request = new Utils.Clients.WebRequest() + var request = new WebRequest { Url = url, Type = method, @@ -482,58 +451,18 @@ namespace Jackett.Common.Indexers PostData = data, Referer = referer, Headers = headers, - Encoding = Encoding - }; - - if (cookieOverride != null) - request.Cookies = cookieOverride; - var result = await webclient.GetBytes(request); - UpdateCookieHeader(result.Cookies, cookieOverride); - return result; - } - - protected async Task PostDataWithCookies(string url, IEnumerable> data, string cookieOverride = null, string referer = null, Dictionary headers = null, string rawbody = null, bool? emulateBrowser = null) - { - var request = new Utils.Clients.WebRequest() - { - Url = url, - Type = RequestType.POST, - Cookies = cookieOverride ?? CookieHeader, - PostData = data, - Referer = referer, - Headers = headers, RawBody = rawbody, Encoding = Encoding }; if (emulateBrowser.HasValue) request.EmulateBrowser = emulateBrowser.Value; - var result = await webclient.GetString(request); - CheckTrackerDown(result); + var result = await webclient.GetResultAsync(request); + CheckSiteDown(result); UpdateCookieHeader(result.Cookies, cookieOverride); return result; } - protected async Task PostDataWithCookiesAndRetry(string url, IEnumerable> data, string cookieOverride = null, string referer = null, Dictionary headers = null, string rawbody = null, bool? emulateBrowser = null) - { - Exception lastException = null; - for (var i = 0; i < 3; i++) - { - try - { - return await PostDataWithCookies(url, data, cookieOverride, referer, headers, rawbody, emulateBrowser); - } - catch (Exception e) - { - logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message)); - lastException = e; - } - await Task.Delay(500); - } - - throw lastException; - } - protected async Task RequestLoginAndFollowRedirect(string url, IEnumerable> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null, bool accumulateCookies = false) { var request = new Utils.Clients.WebRequest() @@ -545,8 +474,8 @@ namespace Jackett.Common.Indexers PostData = data, Encoding = Encoding }; - var response = await webclient.GetString(request); - CheckTrackerDown(response); + var response = await webclient.GetResultAsync(request); + CheckSiteDown(response); if (accumulateCookies) { response.Cookies = ResolveCookies((request.Cookies == null ? "" : request.Cookies + " ") + response.Cookies); @@ -566,7 +495,7 @@ namespace Jackett.Common.Indexers return response; } - protected void CheckTrackerDown(WebResult response) + protected static void CheckSiteDown(WebResult response) { if (response.Status == System.Net.HttpStatusCode.BadGateway || response.Status == System.Net.HttpStatusCode.GatewayTimeout @@ -643,7 +572,7 @@ namespace Jackett.Common.Indexers redirRequestCookies = (overrideCookies != null ? overrideCookies : ""); } // Do redirect - var redirectedResponse = await webclient.GetBytes(new WebRequest() + var redirectedResponse = await webclient.GetResultAsync(new WebRequest() { Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo, Referer = referrer, diff --git a/src/Jackett.Common/Indexers/BitCityReloaded.cs b/src/Jackett.Common/Indexers/BitCityReloaded.cs index 6dad73997..8f456320b 100644 --- a/src/Jackett.Common/Indexers/BitCityReloaded.cs +++ b/src/Jackett.Common/Indexers/BitCityReloaded.cs @@ -129,7 +129,7 @@ namespace Jackett.Common.Indexers searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: BrowseUrl); var results = response.ContentString; try { diff --git a/src/Jackett.Common/Indexers/BitHDTV.cs b/src/Jackett.Common/Indexers/BitHDTV.cs index 9a047fd63..b34808e6a 100644 --- a/src/Jackett.Common/Indexers/BitHDTV.cs +++ b/src/Jackett.Common/Indexers/BitHDTV.cs @@ -63,7 +63,7 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { var result = configData; - var loginPage = await RequestStringWithCookies(LoginUrl, configData.CookieHeader.Value); + var loginPage = await WebRequestWithCookiesAsync(LoginUrl, configData.CookieHeader.Value); if (loginPage.IsRedirect) return result; // already logged in var parser = new HtmlParser(); @@ -129,10 +129,10 @@ namespace Jackett.Common.Indexers qc.Add("search", query.ImdbID); qc.Add("options", "4"); //Search URL field for IMDB link search.Query = qc.GetQueryString(); - results.Add(await RequestStringWithCookiesAndRetry(search.ToString())); + results.Add(await RequestWithCookiesAndRetryAsync(search.ToString())); qc["Options"] = "1"; //Search Title and Description search.Query = qc.GetQueryString(); - results.Add(await RequestStringWithCookiesAndRetry(search.ToString())); + results.Add(await RequestWithCookiesAndRetryAsync(search.ToString())); } else { @@ -140,7 +140,7 @@ namespace Jackett.Common.Indexers qc.Add("search", query.GetQueryString()); qc.Add("options", "0"); //Search Title Only search.Query = qc.GetQueryString(); - results.Add(await RequestStringWithCookiesAndRetry(search.ToString())); + results.Add(await RequestWithCookiesAndRetryAsync(search.ToString())); } var parser = new HtmlParser(); diff --git a/src/Jackett.Common/Indexers/BroadcasTheNet.cs b/src/Jackett.Common/Indexers/BroadcasTheNet.cs index 982d3068a..ba28f0c67 100644 --- a/src/Jackett.Common/Indexers/BroadcasTheNet.cs +++ b/src/Jackett.Common/Indexers/BroadcasTheNet.cs @@ -110,11 +110,13 @@ namespace Jackett.Common.Indexers new JValue(btnResults), new JValue(btnOffset) }; - var response = await PostDataWithCookiesAndRetry(APIBASE, null, null, null, new Dictionary() - { - { "Accept", "application/json-rpc, application/json"}, - {"Content-Type", "application/json-rpc"} - }, JsonRPCRequest("getTorrents", parameters), false); + var response = await RequestWithCookiesAndRetryAsync( + APIBASE, method: RequestType.POST, + headers: new Dictionary + { + {"Accept", "application/json-rpc, application/json"}, + {"Content-Type", "application/json-rpc"} + }, rawbody: JsonRPCRequest("getTorrents", parameters), emulateBrowser: false); try { diff --git a/src/Jackett.Common/Indexers/CardigannIndexer.cs b/src/Jackett.Common/Indexers/CardigannIndexer.cs index df732db6c..b29d5de58 100644 --- a/src/Jackett.Common/Indexers/CardigannIndexer.cs +++ b/src/Jackett.Common/Indexers/CardigannIndexer.cs @@ -532,13 +532,12 @@ namespace Jackett.Common.Indexers ["g-recaptcha-response"] = CaptchaConfigItem.Value }; var ClearanceUrl = resolvePath("/cdn-cgi/l/chk_captcha?" + CloudFlareQueryCollection.GetQueryString()); - - var ClearanceResult = await RequestStringWithCookies(ClearanceUrl.ToString(), null, SiteLink); + var ClearanceResult = await WebRequestWithCookiesAsync(ClearanceUrl.ToString(), referer: SiteLink); if (ClearanceResult.IsRedirect) // clearance successfull { // request real login page again - landingResult = await RequestStringWithCookies(LoginUrl, null, SiteLink); + landingResult = await WebRequestWithCookiesAsync(LoginUrl, referer: SiteLink); var htmlParser = new HtmlParser(); landingResultDocument = htmlParser.ParseDocument(landingResult.ContentString); } @@ -654,7 +653,7 @@ namespace Jackett.Common.Indexers if (simpleCaptchaPresent != null) { var captchaUrl = resolvePath("simpleCaptcha.php?numImages=1"); - var simpleCaptchaResult = await RequestStringWithCookies(captchaUrl.ToString(), null, LoginUrl); + var simpleCaptchaResult = await WebRequestWithCookiesAsync(captchaUrl.ToString(), referer: LoginUrl); var simpleCaptchaJSON = JObject.Parse(simpleCaptchaResult.ContentString); var captchaSelection = simpleCaptchaJSON["images"][0]["hash"].ToString(); pairs["captchaSelection"] = captchaSelection; @@ -723,7 +722,9 @@ namespace Jackett.Common.Indexers headers.Add("Content-Type", "multipart/form-data; boundary=" + boundary); var body = string.Join("\r\n", bodyParts); - loginResult = await PostDataWithCookies(submitUrl.ToString(), pairs, configData.CookieHeader.Value, SiteLink, headers, body); + loginResult = await WebRequestWithCookiesAsync( + submitUrl.ToString(), configData.CookieHeader.Value, RequestType.POST, SiteLink, pairs, headers, + body); } else { @@ -749,7 +750,7 @@ namespace Jackett.Common.Indexers var LoginUrl = resolvePath(Login.Path + "?" + queryCollection.GetQueryString()).ToString(); configData.CookieHeader.Value = null; - var loginResult = await RequestStringWithCookies(LoginUrl, null, SiteLink); + var loginResult = await WebRequestWithCookiesAsync(LoginUrl, referer: SiteLink); configData.CookieHeader.Value = loginResult.Cookies; checkForError(loginResult, Definition.Login.Error); @@ -759,7 +760,7 @@ namespace Jackett.Common.Indexers var OneUrl = applyGoTemplateText(Definition.Login.Inputs["oneurl"]); var LoginUrl = resolvePath(Login.Path + OneUrl).ToString(); configData.CookieHeader.Value = null; - var loginResult = await RequestStringWithCookies(LoginUrl, null, SiteLink); + var loginResult = await WebRequestWithCookiesAsync(LoginUrl, referer: SiteLink); configData.CookieHeader.Value = loginResult.Cookies; checkForError(loginResult, Definition.Login.Error); @@ -793,7 +794,7 @@ namespace Jackett.Common.Indexers // test if login was successful var LoginTestUrl = resolvePath(Login.Test.Path).ToString(); - var testResult = await RequestStringWithCookies(LoginTestUrl); + var testResult = await WebRequestWithCookiesAsync(LoginTestUrl); if (testResult.IsRedirect) { @@ -886,7 +887,7 @@ namespace Jackett.Common.Indexers configData.CookieHeader.Value = null; if (Login.Cookies != null) configData.CookieHeader.Value = string.Join("; ", Login.Cookies); - landingResult = await RequestStringWithCookies(LoginUrl.AbsoluteUri, null, SiteLink); + landingResult = await WebRequestWithCookiesAsync(LoginUrl.AbsoluteUri, referer: SiteLink); var htmlParser = new HtmlParser(); landingResultDocument = htmlParser.ParseDocument(landingResult.ContentString); @@ -929,7 +930,8 @@ namespace Jackett.Common.Indexers hasCaptcha = true; var CaptchaUrl = resolvePath(captchaElement.GetAttribute("src"), LoginUrl); - var captchaImageData = await RequestBytesWithCookies(CaptchaUrl.ToString(), landingResult.Cookies, RequestType.GET, LoginUrl.AbsoluteUri); + var captchaImageData = await WebRequestWithCookiesAsync( + CaptchaUrl.ToString(), landingResult.Cookies, referer: LoginUrl.AbsoluteUri); var CaptchaImage = new ImageItem { Name = "Captcha Image" }; var CaptchaText = new StringItem { Name = "Captcha Text" }; @@ -1334,7 +1336,6 @@ namespace Jackett.Common.Indexers logger.Info($"Fetching: {searchUrl}"); // send HTTP request - WebResult response = null; Dictionary headers = null; if (Search.Headers != null) { @@ -1343,10 +1344,9 @@ namespace Jackett.Common.Indexers foreach (var header in Search.Headers) headers.Add(header.Key, header.Value[0]); } - if (method == RequestType.POST) - response = await PostDataWithCookies(searchUrl, queryCollection, null, null, headers); - else - response = await RequestStringWithCookies(searchUrl, null, null, headers); + + var response = await WebRequestWithCookiesAsync( + searchUrl, method: method, headers: headers, data: queryCollection); if (response.IsRedirect && SearchPath.Followredirect) await FollowIfRedirect(response); @@ -1368,11 +1368,7 @@ namespace Jackett.Common.Indexers if (!LoginResult) throw new Exception(string.Format("Relogin failed")); await TestLogin(); - if (method == RequestType.POST) - response = await PostDataWithCookies(searchUrl, queryCollection); - else - response = await RequestStringWithCookies(searchUrl); - + response = await WebRequestWithCookiesAsync(searchUrl, method: method, data: queryCollection); if (response.IsRedirect && SearchPath.Followredirect) await FollowIfRedirect(response); @@ -1753,7 +1749,7 @@ namespace Jackett.Common.Indexers requestLinkStr += queryCollection.GetQueryString(Encoding, separator: request.Queryseparator); } - var response = await RequestBytesWithCookiesAndRetry(requestLinkStr, null, method, referer, pairs); + var response = await RequestWithCookiesAndRetryAsync(requestLinkStr, null, method, referer, pairs); logger.Debug($"CardigannIndexer ({Id}): handleRequest() remote server returned {response.Status.ToString()}" + (response.IsRedirect ? " => " + response.RedirectingTo : "")); return response; } @@ -1796,9 +1792,9 @@ namespace Jackett.Common.Indexers if (Download.Selector != null) { var selector = applyGoTemplateText(Download.Selector, variables); - var response = await RequestStringWithCookies(link.ToString()); + var response = await WebRequestWithCookiesAsync(link.ToString()); if (response.IsRedirect) - response = await RequestStringWithCookies(response.RedirectingTo); + response = await WebRequestWithCookiesAsync(response.RedirectingTo); var results = response.ContentString; var searchResultParser = new HtmlParser(); var searchResultDocument = searchResultParser.ParseDocument(results); diff --git a/src/Jackett.Common/Indexers/Cinecalidad.cs b/src/Jackett.Common/Indexers/Cinecalidad.cs index 02b829ddf..9dfb654f7 100644 --- a/src/Jackett.Common/Indexers/Cinecalidad.cs +++ b/src/Jackett.Common/Indexers/Cinecalidad.cs @@ -100,7 +100,7 @@ namespace Jackett.Common.Indexers { var pageParam = page > 1 ? $"page/{page}/" : ""; var searchUrl = string.Format(templateUrl, pageParam); - var response = await RequestStringWithCookiesAndRetry(searchUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl); var pageReleases = ParseReleases(response, query); // publish date is not available in the torrent list, but we add a relative date so we can sort @@ -120,7 +120,7 @@ namespace Jackett.Common.Indexers public override async Task Download(Uri link) { - var results = await RequestStringWithCookies(link.ToString()); + var results = await WebRequestWithCookiesAsync(link.ToString()); try { @@ -129,7 +129,7 @@ namespace Jackett.Common.Indexers var preotectedLink = dom.QuerySelector("a[service=BitTorrent]").GetAttribute("href"); preotectedLink = SiteLink + preotectedLink.TrimStart('/'); - results = await RequestStringWithCookies(preotectedLink); + results = await WebRequestWithCookiesAsync(preotectedLink); dom = parser.ParseDocument(results.ContentString); var magnetUrl = dom.QuerySelector("a[href^=magnet]").GetAttribute("href"); return await base.Download(new Uri(magnetUrl)); diff --git a/src/Jackett.Common/Indexers/CorsaroRed.cs b/src/Jackett.Common/Indexers/CorsaroRed.cs index 54f641591..b87cc600a 100644 --- a/src/Jackett.Common/Indexers/CorsaroRed.cs +++ b/src/Jackett.Common/Indexers/CorsaroRed.cs @@ -130,13 +130,14 @@ namespace Jackett.Common.Indexers private async Task SendApiRequest(IEnumerable> data) { - var result = await PostDataWithCookiesAndRetry(ApiSearch, data, null, SiteLink, _apiHeaders, null, true); + var result = await RequestWithCookiesAndRetryAsync( + ApiSearch, null, RequestType.POST, SiteLink, data, _apiHeaders, null, true); return CheckResponse(result); } private async Task SendApiRequestLatest() { - var result = await RequestStringWithCookiesAndRetry(ApiLatest, null, SiteLink, _apiHeaders); + var result = await RequestWithCookiesAndRetryAsync(ApiLatest, referer: SiteLink, headers: _apiHeaders); return CheckResponse(result); } diff --git a/src/Jackett.Common/Indexers/DanishBits.cs b/src/Jackett.Common/Indexers/DanishBits.cs index 342a0ed97..14343eefa 100644 --- a/src/Jackett.Common/Indexers/DanishBits.cs +++ b/src/Jackett.Common/Indexers/DanishBits.cs @@ -65,10 +65,13 @@ namespace Jackett.Common.Indexers return searchString; } - protected override async Task RequestBytesWithCookies(string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, IEnumerable> data = null, Dictionary headers = null) + protected override async Task WebRequestWithCookiesAsync( + string url, string cookieOverride = null, RequestType method = RequestType.GET, string referer = null, + IEnumerable> data = null, Dictionary headers = null, + string rawbody = null, bool? emulateBrowser = null) { CookieHeader = null; // Download fill fail with cookies set - return await base.RequestBytesWithCookies(url, cookieOverride, method, referer, data, headers); + return await base.WebRequestWithCookiesAsync(url, cookieOverride, method, referer, data, headers); } } } diff --git a/src/Jackett.Common/Indexers/DigitalCore.cs b/src/Jackett.Common/Indexers/DigitalCore.cs index 27b327e08..ea016a4e4 100644 --- a/src/Jackett.Common/Indexers/DigitalCore.cs +++ b/src/Jackett.Common/Indexers/DigitalCore.cs @@ -136,7 +136,7 @@ namespace Jackett.Common.Indexers searchUrl += "?" + queryCollection.GetQueryString(); foreach (var cat in MapTorznabCapsToTrackers(query)) searchUrl += "&categories[]=" + cat; - var results = await RequestStringWithCookies(searchUrl, null, SiteLink); + var results = await WebRequestWithCookiesAsync(searchUrl, referer: SiteLink); try { diff --git a/src/Jackett.Common/Indexers/DigitalHive.cs b/src/Jackett.Common/Indexers/DigitalHive.cs index e1c9d67da..c1bd3672c 100644 --- a/src/Jackett.Common/Indexers/DigitalHive.cs +++ b/src/Jackett.Common/Indexers/DigitalHive.cs @@ -93,7 +93,7 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { - var loginPage = await RequestStringWithCookies(LoginUrl, configData.CookieHeader.Value); + var loginPage = await WebRequestWithCookiesAsync(LoginUrl, configData.CookieHeader.Value); var parser = new HtmlParser(); var cq = parser.ParseDocument(loginPage.ContentString); var recaptchaSiteKey = cq.QuerySelector(".g-recaptcha")?.GetAttribute("data-sitekey"); @@ -181,12 +181,12 @@ namespace Jackett.Common.Indexers queryCollection.Add("blah", "0"); - var results = await RequestStringWithCookiesAndRetry(searchUrl + "?" + queryCollection.GetQueryString()); + var results = await RequestWithCookiesAndRetryAsync(searchUrl + "?" + queryCollection.GetQueryString()); if (results.IsRedirect) { // re-login await ApplyConfiguration(null); - results = await RequestStringWithCookiesAndRetry(searchUrl + "?" + queryCollection.GetQueryString()); + results = await RequestWithCookiesAndRetryAsync(searchUrl + "?" + queryCollection.GetQueryString()); } try { diff --git a/src/Jackett.Common/Indexers/DivxTotal.cs b/src/Jackett.Common/Indexers/DivxTotal.cs index ba92496a9..097df53f8 100644 --- a/src/Jackett.Common/Indexers/DivxTotal.cs +++ b/src/Jackett.Common/Indexers/DivxTotal.cs @@ -100,7 +100,7 @@ namespace Jackett.Common.Indexers do { var url = SiteLink + "page/" + page + "/?" + qc.GetQueryString(); - var result = await RequestStringWithCookies(url); + var result = await WebRequestWithCookiesAsync(url); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); @@ -153,7 +153,7 @@ namespace Jackett.Common.Indexers // for other categories we have to do another step if (!downloadUrl.Contains(DownloadLink)) { - var result = await RequestStringWithCookies(downloadUrl); + var result = await WebRequestWithCookiesAsync(downloadUrl); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); @@ -205,7 +205,7 @@ namespace Jackett.Common.Indexers private async Task ParseSeriesRelease(ICollection releases, TorznabQuery query, string commentsLink, string cat, DateTime publishDate) { - var result = await RequestStringWithCookies(commentsLink); + var result = await WebRequestWithCookiesAsync(commentsLink); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); diff --git a/src/Jackett.Common/Indexers/EliteTracker.cs b/src/Jackett.Common/Indexers/EliteTracker.cs index 629748796..51091b265 100644 --- a/src/Jackett.Common/Indexers/EliteTracker.cs +++ b/src/Jackett.Common/Indexers/EliteTracker.cs @@ -151,7 +151,7 @@ namespace Jackett.Common.Indexers { "password", configData.Password.Value } }; - var result = await PostDataWithCookies(LoginUrl, pairs, ""); + var result = await WebRequestWithCookiesAsync(LoginUrl, "", RequestType.POST, data: pairs); await ConfigureIfOK(result.Cookies, result.Cookies != null, () => { @@ -174,12 +174,12 @@ namespace Jackett.Common.Indexers {"category", "0"} // multi cat search not supported }; - var results = await PostDataWithCookies(BrowseUrl, pairs); + var results = await WebRequestWithCookiesAsync(BrowseUrl, method: RequestType.POST, data: pairs); if (results.IsRedirect) { // re-login await ApplyConfiguration(null); - results = await PostDataWithCookies(BrowseUrl, pairs); + results = await WebRequestWithCookiesAsync(BrowseUrl, method: RequestType.POST, data: pairs); } try diff --git a/src/Jackett.Common/Indexers/EpubLibre.cs b/src/Jackett.Common/Indexers/EpubLibre.cs index df1a96dff..feca335c6 100644 --- a/src/Jackett.Common/Indexers/EpubLibre.cs +++ b/src/Jackett.Common/Indexers/EpubLibre.cs @@ -9,6 +9,7 @@ using AngleSharp.Html.Parser; using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; @@ -85,7 +86,7 @@ namespace Jackett.Common.Indexers for (var page = 0; page < maxPages; page++) { var searchUrl = string.Format(SearchUrl, page * MaxItemsPerPage, searchString); - var result = await RequestStringWithCookies(searchUrl, null, null, _apiHeaders); + var result = await WebRequestWithCookiesAsync(searchUrl, headers: _apiHeaders); try { @@ -149,7 +150,7 @@ namespace Jackett.Common.Indexers public override async Task Download(Uri link) { - var result = await RequestStringWithCookiesAndRetry(link.AbsoluteUri); + var result = await RequestWithCookiesAndRetryAsync(link.AbsoluteUri); if (SobrecargaUrl.Equals(result.RedirectingTo)) throw new Exception("El servidor se encuentra sobrecargado en estos momentos. / The server is currently overloaded."); try diff --git a/src/Jackett.Common/Indexers/Feeds/BaseFeedIndexer.cs b/src/Jackett.Common/Indexers/Feeds/BaseFeedIndexer.cs index 89c1b224c..a4a8e9b1b 100644 --- a/src/Jackett.Common/Indexers/Feeds/BaseFeedIndexer.cs +++ b/src/Jackett.Common/Indexers/Feeds/BaseFeedIndexer.cs @@ -55,7 +55,7 @@ namespace Jackett.Common.Indexers.Feeds Type = RequestType.GET, Encoding = Encoding }; - var result = await webclient.GetString(request); + var result = await webclient.GetResultAsync(request); var results = ParseFeedForResults(result.ContentString); diff --git a/src/Jackett.Common/Indexers/FileList.cs b/src/Jackett.Common/Indexers/FileList.cs index 75a110e41..d0cdf8107 100644 --- a/src/Jackett.Common/Indexers/FileList.cs +++ b/src/Jackett.Common/Indexers/FileList.cs @@ -188,7 +188,7 @@ namespace Jackett.Common.Indexers { {"Authorization", "Basic " + auth} }; - var response = await RequestStringWithCookies(searchUrl, headers: headers); + var response = await WebRequestWithCookiesAsync(searchUrl, headers: headers); return response.ContentString; } catch (Exception inner) diff --git a/src/Jackett.Common/Indexers/FunFile.cs b/src/Jackett.Common/Indexers/FunFile.cs index 04a20fdee..c6d9aaa78 100644 --- a/src/Jackett.Common/Indexers/FunFile.cs +++ b/src/Jackett.Common/Indexers/FunFile.cs @@ -97,12 +97,12 @@ namespace Jackett.Common.Indexers qc.Add("search", query.GetQueryString()); var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var results = await RequestStringWithCookiesAndRetry(searchUrl); + var results = await RequestWithCookiesAndRetryAsync(searchUrl); if (results.IsRedirect) // re-login { await ApplyConfiguration(null); - results = await RequestStringWithCookiesAndRetry(searchUrl); + results = await RequestWithCookiesAndRetryAsync(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/Fuzer.cs b/src/Jackett.Common/Indexers/Fuzer.cs index cc01934aa..2e58bdb38 100644 --- a/src/Jackett.Common/Indexers/Fuzer.cs +++ b/src/Jackett.Common/Indexers/Fuzer.cs @@ -13,6 +13,7 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -107,7 +108,7 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { - var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); + var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); var parser = new HtmlParser(); var cq = parser.ParseDocument(loginPage.ContentString); var captcha = cq.QuerySelector(".g-recaptcha"); // invisible recaptcha @@ -153,7 +154,7 @@ namespace Jackett.Common.Indexers } } - var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); + var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); var pairs = new Dictionary { {"vb_login_username", configData.Username.Value}, @@ -204,7 +205,7 @@ namespace Jackett.Common.Indexers } searchUrl = MapTorznabCapsToTrackers(query).Aggregate(searchUrl, (current, cat) => $"{current}&c[]={cat}"); - var data = await RequestStringWithCookiesAndRetry(searchUrl); + var data = await RequestWithCookiesAndRetryAsync(searchUrl); try { var parser = new HtmlParser(); @@ -276,7 +277,7 @@ namespace Jackett.Common.Indexers Path = "index.php", Query = queryString.GetQueryString() }; - var results = await RequestStringWithCookies(site.ToString()); + var results = await WebRequestWithCookiesAsync(site.ToString()); var parser = new HtmlParser(); var dom = parser.ParseDocument(results.ContentString); var rows = dom.QuerySelectorAll("#listtable > tbody > tr"); diff --git a/src/Jackett.Common/Indexers/GazelleGames.cs b/src/Jackett.Common/Indexers/GazelleGames.cs index b683a47e8..8b4f80935 100644 --- a/src/Jackett.Common/Indexers/GazelleGames.cs +++ b/src/Jackett.Common/Indexers/GazelleGames.cs @@ -225,7 +225,7 @@ namespace Jackett.Common.Indexers searchUrl += "?" + queryCollection.GetQueryString(); - var results = await RequestStringWithCookies(searchUrl); + var results = await WebRequestWithCookiesAsync(searchUrl); if (results.IsRedirect && results.RedirectingTo.EndsWith("login.php")) { throw new Exception("relogin needed, please update your cookie"); diff --git a/src/Jackett.Common/Indexers/GimmePeers.cs b/src/Jackett.Common/Indexers/GimmePeers.cs index 415aa424b..943ad2d22 100644 --- a/src/Jackett.Common/Indexers/GimmePeers.cs +++ b/src/Jackett.Common/Indexers/GimmePeers.cs @@ -88,7 +88,7 @@ namespace Jackett.Common.Indexers { "login", "Log in!" } }; - var loginPage = await RequestStringWithCookies(SiteLink, string.Empty); + var loginPage = await WebRequestWithCookiesAsync(SiteLink, string.Empty); var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, SiteLink, SiteLink); await ConfigureIfOK(result.Cookies, result.ContentString?.Contains("logout.php") == true, () => @@ -123,12 +123,12 @@ namespace Jackett.Common.Indexers } searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: BrowseUrl); if (response.IsRedirect) { // re login await ApplyConfiguration(null); - response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl); + response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: BrowseUrl); } var results = response.ContentString; diff --git a/src/Jackett.Common/Indexers/HDBitsApi.cs b/src/Jackett.Common/Indexers/HDBitsApi.cs index 478f4bd6e..f6d4f8986 100644 --- a/src/Jackett.Common/Indexers/HDBitsApi.cs +++ b/src/Jackett.Common/Indexers/HDBitsApi.cs @@ -176,14 +176,14 @@ namespace Jackett.Common.Indexers requestData["username"] = configData.Username.Value; requestData["passkey"] = configData.Passkey.Value; JObject json = null; - - var response = await PostDataWithCookiesAndRetry(APIUrl + url, null, null, null, new Dictionary() - { - {"Accept", "application/json"}, - {"Content-Type", "application/json"} - }, requestData.ToString(), false); - - CheckTrackerDown(response); + var response = await RequestWithCookiesAndRetryAsync( + APIUrl + url, null, RequestType.POST, null, null, + new Dictionary + { + {"Accept", "application/json"}, + {"Content-Type", "application/json"} + }, requestData.ToString(), false); + CheckSiteDown(response); try { diff --git a/src/Jackett.Common/Indexers/HDSpace.cs b/src/Jackett.Common/Indexers/HDSpace.cs index 4cb97ed81..549fa49bb 100644 --- a/src/Jackett.Common/Indexers/HDSpace.cs +++ b/src/Jackett.Common/Indexers/HDSpace.cs @@ -77,7 +77,7 @@ namespace Jackett.Common.Indexers public override async Task ApplyConfiguration(JToken configJson) { LoadValuesFromJson(configJson); - var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); + var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); var pairs = new Dictionary { {"uid", configData.Username.Value}, @@ -119,7 +119,7 @@ namespace Jackett.Common.Indexers queryCollection.Add("search", query.GetQueryString()); } - var response = await RequestStringWithCookiesAndRetry(SearchUrl + queryCollection.GetQueryString()); + var response = await RequestWithCookiesAndRetryAsync(SearchUrl + queryCollection.GetQueryString()); try { diff --git a/src/Jackett.Common/Indexers/HDTorrents.cs b/src/Jackett.Common/Indexers/HDTorrents.cs index 705a47a42..39a046950 100644 --- a/src/Jackett.Common/Indexers/HDTorrents.cs +++ b/src/Jackett.Common/Indexers/HDTorrents.cs @@ -104,7 +104,7 @@ namespace Jackett.Common.Indexers { LoadValuesFromJson(configJson); - var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); + var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); var pairs = new Dictionary { { "uid", configData.Username.Value }, @@ -134,7 +134,7 @@ namespace Jackett.Common.Indexers // manually url encode parenthesis to prevent "hacking" detection searchUrl += queryCollection.GetQueryString().Replace("(", "%28").Replace(")", "%29"); - var results = await RequestStringWithCookiesAndRetry(searchUrl); + var results = await RequestWithCookiesAndRetryAsync(searchUrl); try { var parser = new HtmlParser(); diff --git a/src/Jackett.Common/Indexers/Hebits.cs b/src/Jackett.Common/Indexers/Hebits.cs index cea44ac72..0d329112e 100644 --- a/src/Jackett.Common/Indexers/Hebits.cs +++ b/src/Jackett.Common/Indexers/Hebits.cs @@ -105,7 +105,7 @@ namespace Jackett.Common.Indexers var cats = MapTorznabCapsToTrackers(query); if (cats.Count > 0) searchUrl = cats.Aggregate(searchUrl, (url, cat) => $"{url}&c{cat}=1"); - var response = await RequestStringWithCookies(searchUrl); + var response = await WebRequestWithCookiesAsync(searchUrl); try { var parser = new HtmlParser(); diff --git a/src/Jackett.Common/Indexers/HorribleSubs.cs b/src/Jackett.Common/Indexers/HorribleSubs.cs index 89815d95c..7fae39625 100644 --- a/src/Jackett.Common/Indexers/HorribleSubs.cs +++ b/src/Jackett.Common/Indexers/HorribleSubs.cs @@ -72,7 +72,7 @@ namespace Jackett.Common.Indexers }; var searchUrl = ApiEndpoint + "?" + queryCollection.GetQueryString(); - var response = await RequestStringWithCookiesAndRetry(searchUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl); try { @@ -112,7 +112,7 @@ namespace Jackett.Common.Indexers }; var searchUrl = ApiEndpoint + "?" + queryCollection.GetQueryString(); - var response = await RequestStringWithCookiesAndRetry(searchUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl); try { @@ -146,7 +146,7 @@ namespace Jackett.Common.Indexers var releases = new List(); var parser = new HtmlParser(); - var response = await RequestStringWithCookiesAndRetry(resultUrl); + var response = await RequestWithCookiesAndRetryAsync(resultUrl); await FollowIfRedirect(response); try @@ -168,7 +168,7 @@ namespace Jackett.Common.Indexers var nextId = 0; while (true) { - var showApiResponse = await RequestStringWithCookiesAndRetry(apiUrl + "&nextid=" + nextId); + var showApiResponse = await RequestWithCookiesAndRetryAsync(apiUrl + "&nextid=" + nextId); var showApiDom = parser.ParseDocument(showApiResponse.ContentString); var releaseRowResults = showApiDom.QuerySelectorAll("div.rls-info-container"); rows.AddRange(releaseRowResults); diff --git a/src/Jackett.Common/Indexers/IPTorrents.cs b/src/Jackett.Common/Indexers/IPTorrents.cs index 5c71c3be4..3fd4fd004 100644 --- a/src/Jackett.Common/Indexers/IPTorrents.cs +++ b/src/Jackett.Common/Indexers/IPTorrents.cs @@ -10,6 +10,7 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -179,7 +180,7 @@ namespace Jackett.Common.Indexers qc.Add(cat, string.Empty); var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var response = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl); var results = response.ContentString; if (results == null || !results.Contains("/lout.php")) diff --git a/src/Jackett.Common/Indexers/ImmortalSeed.cs b/src/Jackett.Common/Indexers/ImmortalSeed.cs index 819b6d495..c81f41083 100644 --- a/src/Jackett.Common/Indexers/ImmortalSeed.cs +++ b/src/Jackett.Common/Indexers/ImmortalSeed.cs @@ -11,6 +11,7 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -121,13 +122,13 @@ namespace Jackett.Common.Indexers if (!string.IsNullOrWhiteSpace(query.GetQueryString())) searchUrl += string.Format(QueryString, WebUtility.UrlEncode(query.GetQueryString())); - var results = await RequestStringWithCookiesAndRetry(searchUrl); + var results = await RequestWithCookiesAndRetryAsync(searchUrl); // Occasionally the cookies become invalid, login again if that happens if (results.ContentString.Contains("You do not have permission to access this page.")) { await ApplyConfiguration(null); - results = await RequestStringWithCookiesAndRetry(searchUrl); + results = await RequestWithCookiesAndRetryAsync(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/InternetArchive.cs b/src/Jackett.Common/Indexers/InternetArchive.cs index 62600891b..2557d0636 100644 --- a/src/Jackett.Common/Indexers/InternetArchive.cs +++ b/src/Jackett.Common/Indexers/InternetArchive.cs @@ -143,7 +143,7 @@ namespace Jackett.Common.Indexers {"output", "json"} }; var fullSearchUrl = SearchUrl + "?" + qc.GetQueryString(); - var result = await RequestStringWithCookiesAndRetry(fullSearchUrl); + var result = await RequestWithCookiesAndRetryAsync(fullSearchUrl); foreach (var torrent in ParseResponse(result)) releases.Add(MakeRelease(torrent)); diff --git a/src/Jackett.Common/Indexers/Libble.cs b/src/Jackett.Common/Indexers/Libble.cs index 3aa7931dc..105a6c31a 100644 --- a/src/Jackett.Common/Indexers/Libble.cs +++ b/src/Jackett.Common/Indexers/Libble.cs @@ -83,7 +83,7 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { - await RequestStringWithCookies(LandingUrl); + await WebRequestWithCookiesAsync(LandingUrl); return configData; } @@ -155,12 +155,12 @@ namespace Jackett.Common.Indexers searchUrl += "?" + queryCollection.GetQueryString(); - var searchPage = await PostDataWithCookiesAndRetry(searchUrl, searchParams); + var searchPage = await RequestWithCookiesAndRetryAsync(searchUrl, method: RequestType.POST, data: searchParams); // Occasionally the cookies become invalid, login again if that happens if (searchPage.IsRedirect) { await ApplyConfiguration(null); - searchPage = await PostDataWithCookiesAndRetry(searchUrl, searchParams); + searchPage = await RequestWithCookiesAndRetryAsync(searchUrl, method: RequestType.POST, data: searchParams); } try diff --git a/src/Jackett.Common/Indexers/LostFilm.cs b/src/Jackett.Common/Indexers/LostFilm.cs index ae496a08e..9aad1eb02 100644 --- a/src/Jackett.Common/Indexers/LostFilm.cs +++ b/src/Jackett.Common/Indexers/LostFilm.cs @@ -113,14 +113,14 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { // looks like after some failed login attempts there's a captcha - var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); + var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); var parser = new HtmlParser(); var document = parser.ParseDocument(loginPage.ContentString); var qCaptchaImg = document.QuerySelector("img#captcha_pictcha"); if (qCaptchaImg != null) { var captchaUrl = SiteLink + qCaptchaImg.GetAttribute("src"); - var captchaImage = await RequestBytesWithCookies(captchaUrl, loginPage.Cookies); + var captchaImage = await WebRequestWithCookiesAsync(captchaUrl, loginPage.Cookies); configData.CaptchaImage.Value = captchaImage.ContentBytes; } else @@ -183,7 +183,7 @@ namespace Jackett.Common.Indexers { "type", "logout" } }; - var response = await PostDataWithCookies(url: ApiUrl, data: data); + var response = await WebRequestWithCookiesAsync(ApiUrl, method: RequestType.POST, data: data); logger.Debug("Logout result: " + response.ContentString); var isOK = response.Status == System.Net.HttpStatusCode.OK; @@ -214,7 +214,7 @@ namespace Jackett.Common.Indexers private async Task RequestStringAndRelogin(string url) { - var results = await RequestStringWithCookies(url); + var results = await WebRequestWithCookiesAsync(url); if (results.ContentString.Contains("503 Service")) { throw new ExceptionWithConfigData(results.ContentString, configData); @@ -223,7 +223,7 @@ namespace Jackett.Common.Indexers { // Re-login await ApplyConfiguration(null); - return await RequestStringWithCookies(url); + return await WebRequestWithCookiesAsync(url); } else { @@ -287,7 +287,7 @@ namespace Jackett.Common.Indexers { "val", searchString } }; logger.Debug("> Searching: " + searchString); - var response = await PostDataWithCookies(url: ApiUrl, data: data); + var response = await WebRequestWithCookiesAsync(ApiUrl, method: RequestType.POST, data: data); if (response.ContentString == null) { logger.Debug("> Empty series response for query: " + searchString); @@ -471,7 +471,7 @@ namespace Jackett.Common.Indexers logger.Debug("FetchSeriesReleases: " + url + " S: " + query.Season.ToString() + " E: " + query.Episode + " Filter: " + filter); var releases = new List(); - var results = await RequestStringWithCookies(url); + var results = await WebRequestWithCookiesAsync(url); try { @@ -626,7 +626,7 @@ namespace Jackett.Common.Indexers logger.Debug("FetchTrackerReleases: " + url); // Get redirection page with generated link on it. This link can't be constructed manually as it contains Hash field and hashing algo is unknown. - var results = await RequestStringWithCookies(url); + var results = await WebRequestWithCookiesAsync(url); if (results.ContentString == null) { throw new ExceptionWithConfigData("Empty response from " + url, configData); @@ -659,7 +659,7 @@ namespace Jackett.Common.Indexers private async Task> FollowTrackerRedirection(string url, TrackerUrlDetails details) { logger.Debug("FollowTrackerRedirection: " + url); - var results = await RequestStringWithCookies(url); + var results = await WebRequestWithCookiesAsync(url); var releases = new List(); try diff --git a/src/Jackett.Common/Indexers/MejorTorrent.cs b/src/Jackett.Common/Indexers/MejorTorrent.cs index 2275e2504..6ad562d8b 100644 --- a/src/Jackett.Common/Indexers/MejorTorrent.cs +++ b/src/Jackett.Common/Indexers/MejorTorrent.cs @@ -101,14 +101,14 @@ namespace Jackett.Common.Indexers var downloadUrl = link.ToString(); // Eg https://www.mejortorrentt.net/peli-descargar-torrent-11995-Harry-Potter-y-la-piedra-filosofal.html - var result = await RequestStringWithCookies(downloadUrl); + var result = await WebRequestWithCookiesAsync(downloadUrl); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); var dom = parser.ParseDocument(result.ContentString); downloadUrl = SiteLink + dom.QuerySelector("a[href*=\"sec=descargas\"]").GetAttribute("href"); // Eg https://www.mejortorrentt.net/secciones.php?sec=descargas&ap=contar&tabla=peliculas&id=11995&link_bajar=1 - result = await RequestStringWithCookies(downloadUrl); + result = await WebRequestWithCookiesAsync(downloadUrl); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); dom = parser.ParseDocument(result.ContentString); @@ -123,7 +123,7 @@ namespace Jackett.Common.Indexers { var releases = new List(); var url = SiteLink + NewTorrentsUrl; - var result = await RequestStringWithCookies(url); + var result = await WebRequestWithCookiesAsync(url); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); try @@ -180,7 +180,7 @@ namespace Jackett.Common.Indexers var searchTerm = GetLongestWord(query.SearchTerm); var qc = new NameValueCollection { { "sec", "buscador" }, { "valor", searchTerm } }; var url = SiteLink + SearchUrl + "?" + qc.GetQueryString(); - var result = await RequestStringWithCookies(url); + var result = await WebRequestWithCookiesAsync(url); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); @@ -259,7 +259,7 @@ namespace Jackett.Common.Indexers private async Task ParseSeriesRelease(ICollection releases, TorznabQuery query, string title, string commentsLink, string cat, DateTime publishDate) { - var result = await RequestStringWithCookies(commentsLink); + var result = await WebRequestWithCookiesAsync(commentsLink); if (result.Status != HttpStatusCode.OK) throw new ExceptionWithConfigData(result.ContentString, configData); diff --git a/src/Jackett.Common/Indexers/Milkie.cs b/src/Jackett.Common/Indexers/Milkie.cs index 385410fee..43cb15b95 100644 --- a/src/Jackett.Common/Indexers/Milkie.cs +++ b/src/Jackett.Common/Indexers/Milkie.cs @@ -87,7 +87,7 @@ namespace Jackett.Common.Indexers { { "x-milkie-auth", configData.Key.Value } }; - var jsonResponse = await RequestStringWithCookies(endpoint, headers: headers); + var jsonResponse = await WebRequestWithCookiesAsync(endpoint, headers: headers); var releases = new List(); diff --git a/src/Jackett.Common/Indexers/MoreThanTV.cs b/src/Jackett.Common/Indexers/MoreThanTV.cs index 57b681fcc..167698207 100644 --- a/src/Jackett.Common/Indexers/MoreThanTV.cs +++ b/src/Jackett.Common/Indexers/MoreThanTV.cs @@ -13,6 +13,7 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -59,8 +60,7 @@ namespace Jackett.Common.Indexers { "login", "Log in" }, { "keeplogged", "1" } }; - - var preRequest = await RequestStringWithCookiesAndRetry(LoginUrl, string.Empty); + var preRequest = await RequestWithCookiesAndRetryAsync(LoginUrl, string.Empty); var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, preRequest.Cookies, true, SearchUrl, SiteLink); await ConfigureIfOK(result.Cookies, result.ContentString != null && result.ContentString.Contains("status\":\"success\""), () => @@ -134,12 +134,12 @@ namespace Jackett.Common.Indexers private async Task GetReleases(ICollection releases, TorznabQuery query, string searchQuery) { var searchUrl = GetTorrentSearchUrl(query, searchQuery); - var response = await RequestStringWithCookiesAndRetry(searchUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl); if (response.IsRedirect) { // re login await ApplyConfiguration(null); - response = await RequestStringWithCookiesAndRetry(searchUrl); + response = await RequestWithCookiesAndRetryAsync(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/MyAmity.cs b/src/Jackett.Common/Indexers/MyAmity.cs index 5eb7d5a9a..0b60daced 100644 --- a/src/Jackett.Common/Indexers/MyAmity.cs +++ b/src/Jackett.Common/Indexers/MyAmity.cs @@ -119,12 +119,12 @@ namespace Jackett.Common.Indexers } searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestStringWithCookies(searchUrl); + var response = await WebRequestWithCookiesAsync(searchUrl); if (response.IsRedirect || response.Cookies != null && response.Cookies.Contains("pass=deleted;")) { // re-login await ApplyConfiguration(null); - response = await RequestStringWithCookies(searchUrl); + response = await WebRequestWithCookiesAsync(searchUrl); } var results = response.ContentString; diff --git a/src/Jackett.Common/Indexers/MyAnonamouse.cs b/src/Jackett.Common/Indexers/MyAnonamouse.cs index 8ef2f44a1..bd2f56422 100644 --- a/src/Jackett.Common/Indexers/MyAnonamouse.cs +++ b/src/Jackett.Common/Indexers/MyAnonamouse.cs @@ -199,7 +199,7 @@ namespace Jackett.Common.Indexers if (qParams.Count > 0) urlSearch += $"?{qParams.GetQueryString()}"; - var response = await RequestStringWithCookiesAndRetry(urlSearch); + var response = await RequestWithCookiesAndRetryAsync(urlSearch); if (response.ContentString.StartsWith("Error")) throw new Exception(response.ContentString); diff --git a/src/Jackett.Common/Indexers/NCore.cs b/src/Jackett.Common/Indexers/NCore.cs index 9d4095d18..f1c4b5443 100644 --- a/src/Jackett.Common/Indexers/NCore.cs +++ b/src/Jackett.Common/Indexers/NCore.cs @@ -99,7 +99,7 @@ namespace Jackett.Common.Indexers LoadValuesFromJson(configJson); if (configData.Hungarian.Value == false && configData.English.Value == false) throw new ExceptionWithConfigData("Please select at least one language.", configData); - var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); + var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); var pairs = new Dictionary { {"nev", configData.Username.Value}, @@ -166,7 +166,8 @@ namespace Jackett.Common.Indexers cats = cats.Except(_languageCats).ToList(); pairs.Add("kivalasztott_tipus[]", string.Join(",", cats)); - var results = await PostDataWithCookiesAndRetry(SearchUrl, pairs.ToEnumerable(true)); + var results = await RequestWithCookiesAndRetryAsync( + SearchUrl, null, RequestType.POST, null, pairs.ToEnumerable(true)); var parser = new HtmlParser(); var dom = parser.ParseDocument(results.ContentString); @@ -199,7 +200,8 @@ namespace Jackett.Common.Indexers for (var page = startPage; page <= pages && releases.Count < limit; page++) { pairs["oldal"] = page.ToString(); - results = await PostDataWithCookiesAndRetry(SearchUrl, pairs.ToEnumerable(true)); + results = await RequestWithCookiesAndRetryAsync( + SearchUrl, null, RequestType.POST, null, pairs.ToEnumerable(true)); releases.AddRange(ParseTorrents(results, episodeString, query, releases.Count, limit, previouslyParsedOnPage)); previouslyParsedOnPage = 0; } diff --git a/src/Jackett.Common/Indexers/Nebulance.cs b/src/Jackett.Common/Indexers/Nebulance.cs index 9a643c681..19255a185 100644 --- a/src/Jackett.Common/Indexers/Nebulance.cs +++ b/src/Jackett.Common/Indexers/Nebulance.cs @@ -11,6 +11,7 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -74,14 +75,14 @@ namespace Jackett.Common.Indexers protected override async Task> PerformQuery(TorznabQuery query) { - var loggedInCheck = await RequestStringWithCookies(SearchUrl); + var loggedInCheck = await WebRequestWithCookiesAsync(SearchUrl); if (!loggedInCheck.ContentString.Contains("logout.php")) // re-login await DoLogin(); // #6413 var url = $"{SearchUrl}&searchtext={WebUtility.UrlEncode(query.GetQueryString())}"; - var response = await RequestStringWithCookiesAndRetry(url); + var response = await RequestWithCookiesAndRetryAsync(url); var releases = ParseResponse(response.ContentString); return releases; diff --git a/src/Jackett.Common/Indexers/NewPCT.cs b/src/Jackett.Common/Indexers/NewPCT.cs index d8c4d869e..6dbc9b776 100644 --- a/src/Jackett.Common/Indexers/NewPCT.cs +++ b/src/Jackett.Common/Indexers/NewPCT.cs @@ -163,7 +163,7 @@ namespace Jackett.Common.Indexers public override async Task Download(Uri linkParam) { - var results = await RequestStringWithCookiesAndRetry(linkParam.AbsoluteUri); + var results = await RequestWithCookiesAndRetryAsync(linkParam.AbsoluteUri); var uriLink = ExtractDownloadUri(results.ContentString, linkParam.AbsoluteUri); if (uriLink == null) @@ -211,7 +211,7 @@ namespace Jackett.Common.Indexers while (pg <= _maxDailyPages) { var pageUrl = SiteLink + string.Format(_dailyUrl, pg); - var results = await RequestStringWithCookiesAndRetry(pageUrl); + var results = await RequestWithCookiesAndRetryAsync(pageUrl); if (results == null || string.IsNullOrEmpty(results.ContentString)) break; @@ -304,13 +304,13 @@ namespace Jackett.Common.Indexers var releases = new List(); // Episodes list - var results = await RequestStringWithCookiesAndRetry(uri.AbsoluteUri); + var results = await RequestWithCookiesAndRetryAsync(uri.AbsoluteUri); var seriesEpisodesUrl = ParseSeriesListContent(results.ContentString, seriesName); // TV serie list if (!string.IsNullOrEmpty(seriesEpisodesUrl)) { - results = await RequestStringWithCookiesAndRetry(seriesEpisodesUrl); + results = await RequestWithCookiesAndRetryAsync(seriesEpisodesUrl); var items = ParseEpisodesListContent(results.ContentString); if (items != null && items.Any()) releases.AddRange(items); @@ -464,7 +464,7 @@ namespace Jackett.Common.Indexers {"pg", pg.ToString()} }; - var results = await PostDataWithCookies(searchJsonUrl, queryCollection); + var results = await WebRequestWithCookiesAsync(searchJsonUrl, method: RequestType.POST, data: queryCollection); var items = ParseSearchJsonContent(results.ContentString, year); if (!items.Any()) break; diff --git a/src/Jackett.Common/Indexers/NewRealWorld.cs b/src/Jackett.Common/Indexers/NewRealWorld.cs index ff8393815..721913354 100644 --- a/src/Jackett.Common/Indexers/NewRealWorld.cs +++ b/src/Jackett.Common/Indexers/NewRealWorld.cs @@ -150,12 +150,12 @@ namespace Jackett.Common.Indexers searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestStringWithCookies(searchUrl); + var response = await WebRequestWithCookiesAsync(searchUrl); if (response.IsRedirect) { // re-login await ApplyConfiguration(null); - response = await RequestStringWithCookies(searchUrl); + response = await WebRequestWithCookiesAsync(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/NorBits.cs b/src/Jackett.Common/Indexers/NorBits.cs index b3935fd03..fd9ff8c30 100644 --- a/src/Jackett.Common/Indexers/NorBits.cs +++ b/src/Jackett.Common/Indexers/NorBits.cs @@ -136,7 +136,7 @@ namespace Jackett.Common.Indexers // Get index page for cookies Output("\nGetting index page (for cookies).. with " + SiteLink); - var indexPage = await webclient.GetString(myIndexRequest); + var indexPage = await webclient.GetResultAsync(myIndexRequest); // Building login form data var pairs = new Dictionary { @@ -158,7 +158,7 @@ namespace Jackett.Common.Indexers // Get login page -- (not used, but simulation needed by tracker security's checks) LatencyNow(); Output("\nGetting login page (user simulation).. with " + LoginUrl); - await webclient.GetString(myRequestLogin); + await webclient.GetResultAsync(myRequestLogin); // Build WebRequest for submitting authentification var request = new Utils.Clients.WebRequest() @@ -175,7 +175,7 @@ namespace Jackett.Common.Indexers // Perform loggin LatencyNow(); Output("\nPerform loggin.. with " + LoginCheckUrl); - var response = await webclient.GetString(request); + var response = await webclient.GetResultAsync(request); // Test if we are logged in await ConfigureIfOK(response.Cookies, response.Cookies != null && response.Cookies.Contains("uid="), () => @@ -204,7 +204,7 @@ namespace Jackett.Common.Indexers { // Checking ... Output("\n-> Checking logged-in state...."); - var loggedInCheck = await RequestStringWithCookies(SearchUrl); + var loggedInCheck = await WebRequestWithCookiesAsync(SearchUrl); if (!loggedInCheck.ContentString.Contains("logout.php")) { // Cookie expired, renew session on provider @@ -261,7 +261,7 @@ namespace Jackett.Common.Indexers var request = BuildQuery(searchTerm, query, searchUrl); // Getting results & Store content - var response = await RequestStringWithCookiesAndRetry(request, ConfigData.CookieHeader.Value); + var response = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value); var parser = new HtmlParser(); var dom = parser.ParseDocument(response.ContentString); @@ -526,7 +526,7 @@ namespace Jackett.Common.Indexers // Request our first page LatencyNow(); - var results = await RequestStringWithCookiesAndRetry(request, ConfigData.CookieHeader.Value, SearchUrl, _emulatedBrowserHeaders); + var results = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value, RequestType.GET, SearchUrl, null, _emulatedBrowserHeaders); // Return results from tracker return results; diff --git a/src/Jackett.Common/Indexers/NordicBits.cs b/src/Jackett.Common/Indexers/NordicBits.cs index 1b6a118f7..6f4f3b6e7 100644 --- a/src/Jackett.Common/Indexers/NordicBits.cs +++ b/src/Jackett.Common/Indexers/NordicBits.cs @@ -191,7 +191,7 @@ namespace Jackett.Common.Indexers // Get index page for cookies Output("\nGetting index page (for cookies).. with " + SiteLink); - var indexPage = await webclient.GetString(myIndexRequest); + var indexPage = await webclient.GetResultAsync(myIndexRequest); // Building login form data var pairs = new Dictionary { @@ -213,7 +213,7 @@ namespace Jackett.Common.Indexers // Get login page -- (not used, but simulation needed by tracker security's checks) LatencyNow(); Output("\nGetting login page (user simulation).. with " + LoginUrl); - await webclient.GetString(myRequestLogin); + await webclient.GetResultAsync(myRequestLogin); // Build WebRequest for submitting authentification var request = new Utils.Clients.WebRequest() @@ -230,7 +230,7 @@ namespace Jackett.Common.Indexers // Perform loggin LatencyNow(); Output("\nPerform loggin.. with " + LoginCheckUrl); - var response = await webclient.GetString(request); + var response = await webclient.GetResultAsync(request); // Test if we are logged in await ConfigureIfOK(response.Cookies, response.Cookies != null && response.Cookies.Contains("uid="), () => @@ -259,7 +259,7 @@ namespace Jackett.Common.Indexers { // Checking ... Output("\n-> Checking logged-in state...."); - var loggedInCheck = await RequestStringWithCookies(SearchUrl); + var loggedInCheck = await WebRequestWithCookiesAsync(SearchUrl); if (!loggedInCheck.ContentString.Contains("logout.php")) { // Cookie expired, renew session on provider @@ -316,7 +316,7 @@ namespace Jackett.Common.Indexers var request = BuildQuery(searchTerm, query, searchUrl); // Getting results & Store content - var response = await RequestStringWithCookiesAndRetry(request, ConfigData.CookieHeader.Value); + var response = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value); var parser = new HtmlParser(); var dom = parser.ParseDocument(response.ContentString); @@ -597,7 +597,7 @@ namespace Jackett.Common.Indexers // Request our first page LatencyNow(); - var results = await RequestStringWithCookiesAndRetry(request, ConfigData.CookieHeader.Value, SearchUrl, _emulatedBrowserHeaders); + var results = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value, RequestType.GET, SearchUrl, null, _emulatedBrowserHeaders); // Return results from tracker return results; diff --git a/src/Jackett.Common/Indexers/Partis.cs b/src/Jackett.Common/Indexers/Partis.cs index 8c4a91c14..a2597125e 100644 --- a/src/Jackett.Common/Indexers/Partis.cs +++ b/src/Jackett.Common/Indexers/Partis.cs @@ -135,13 +135,13 @@ namespace Jackett.Common.Indexers logger.Info(string.Format("Searh URL Partis_: {0}", searchUrl)); // add necessary headers - var heder = new Dictionary + var header = new Dictionary { { "X-requested-with", "XMLHttpRequest" } }; //get results and follow redirect - results = await RequestStringWithCookies(searchUrl, null, SearchUrl, heder); + results = await WebRequestWithCookiesAsync(searchUrl, referer: SearchUrl, headers: header); await FollowIfRedirect(results, null, null, null, true); // are we logged in? @@ -150,7 +150,8 @@ namespace Jackett.Common.Indexers await ApplyConfiguration(null); } // another request with specific query - NEEDED for succesful response - return data - results = await RequestStringWithCookies(SiteLink + "brskaj/?rs=false&offset=0", null, SearchUrl, heder); + results = await WebRequestWithCookiesAsync( + SiteLink + "brskaj/?rs=false&offset=0", referer: SearchUrl, headers: header); await FollowIfRedirect(results, null, null, null, true); // parse results diff --git a/src/Jackett.Common/Indexers/PassThePopcorn.cs b/src/Jackett.Common/Indexers/PassThePopcorn.cs index 9a68a48d1..af10d7701 100644 --- a/src/Jackett.Common/Indexers/PassThePopcorn.cs +++ b/src/Jackett.Common/Indexers/PassThePopcorn.cs @@ -11,6 +11,7 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -115,9 +116,9 @@ namespace Jackett.Common.Indexers { "ApiKey", configData.Key.Value } }; - var results = await RequestStringWithCookiesAndRetry(movieListSearchUrl, headers: authHeaders); + var results = await RequestWithCookiesAndRetryAsync(movieListSearchUrl, headers: authHeaders); if (results.IsRedirect) // untested - results = await RequestStringWithCookiesAndRetry(movieListSearchUrl, headers: authHeaders); + results = await RequestWithCookiesAndRetryAsync(movieListSearchUrl, headers: authHeaders); try { //Iterate over the releases for each movie diff --git a/src/Jackett.Common/Indexers/PirateTheNet.cs b/src/Jackett.Common/Indexers/PirateTheNet.cs index fa54d834f..e7797cdea 100644 --- a/src/Jackett.Common/Indexers/PirateTheNet.cs +++ b/src/Jackett.Common/Indexers/PirateTheNet.cs @@ -76,7 +76,7 @@ namespace Jackett.Common.Indexers LoadValuesFromJson(configJson); CookieHeader = ""; // clear old cookies - var result1 = await RequestStringWithCookies(CaptchaUrl); + var result1 = await WebRequestWithCookiesAsync(CaptchaUrl); var json1 = JObject.Parse(result1.ContentString); var captchaSelection = json1["images"][0]["hash"]; @@ -125,12 +125,12 @@ namespace Jackett.Common.Indexers var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var results = await RequestStringWithCookiesAndRetry(searchUrl); + var results = await RequestWithCookiesAndRetryAsync(searchUrl); if (results.IsRedirect) { // re-login await ApplyConfiguration(null); - results = await RequestStringWithCookiesAndRetry(searchUrl); + results = await RequestWithCookiesAndRetryAsync(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/PixelHD.cs b/src/Jackett.Common/Indexers/PixelHD.cs index 2814aa60d..784e7be5a 100644 --- a/src/Jackett.Common/Indexers/PixelHD.cs +++ b/src/Jackett.Common/Indexers/PixelHD.cs @@ -57,7 +57,7 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { - var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); + var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); var LoginParser = new HtmlParser(); var LoginDocument = LoginParser.ParseDocument(loginPage.ContentString); @@ -69,7 +69,7 @@ namespace Jackett.Common.Indexers var catchaInput = LoginDocument.QuerySelector("input[maxlength=\"6\"]"); input_captcha = catchaInput.GetAttribute("name"); - var captchaImage = await RequestBytesWithCookies(SiteLink + catchaImg.GetAttribute("src"), loginPage.Cookies, RequestType.GET, LoginUrl); + var captchaImage = await WebRequestWithCookiesAsync(SiteLink + catchaImg.GetAttribute("src"), loginPage.Cookies, RequestType.GET, LoginUrl); configData.CaptchaImage.Value = captchaImage.ContentBytes; } else @@ -137,13 +137,13 @@ namespace Jackett.Common.Indexers var searchUrl = BrowseUrl + "?" + queryCollection.GetQueryString(); - var results = await RequestStringWithCookies(searchUrl); + var results = await WebRequestWithCookiesAsync(searchUrl); if (results.IsRedirect) { // re login await GetConfigurationForSetup(); await ApplyConfiguration(null); - results = await RequestStringWithCookies(searchUrl); + results = await WebRequestWithCookiesAsync(searchUrl); } var IMDBRegEx = new Regex(@"tt(\d+)", RegexOptions.Compiled); diff --git a/src/Jackett.Common/Indexers/PolishTracker.cs b/src/Jackett.Common/Indexers/PolishTracker.cs index a035bdac1..4ca0ea8fd 100644 --- a/src/Jackett.Common/Indexers/PolishTracker.cs +++ b/src/Jackett.Common/Indexers/PolishTracker.cs @@ -104,12 +104,12 @@ namespace Jackett.Common.Indexers qc.Add("cat[]", cat); var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var result = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrl); + var result = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl); if (result.IsRedirect) { // re-login await ApplyConfiguration(null); - result = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrl); + result = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl); } if (!result.ContentString.StartsWith("{")) // not JSON => error diff --git a/src/Jackett.Common/Indexers/PornoLab.cs b/src/Jackett.Common/Indexers/PornoLab.cs index d86068011..65e8a12cd 100644 --- a/src/Jackett.Common/Indexers/PornoLab.cs +++ b/src/Jackett.Common/Indexers/PornoLab.cs @@ -183,13 +183,13 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { configData.CookieHeader.Value = null; - var response = await RequestStringWithCookies(LoginUrl); + var response = await WebRequestWithCookiesAsync(LoginUrl); var LoginResultParser = new HtmlParser(); var LoginResultDocument = LoginResultParser.ParseDocument(response.ContentString); var captchaimg = LoginResultDocument.QuerySelector("img[src*=\"/captcha/\"]"); if (captchaimg != null) { - var captchaImage = await RequestBytesWithCookies("https:" + captchaimg.GetAttribute("src")); + var captchaImage = await WebRequestWithCookiesAsync("https:" + captchaimg.GetAttribute("src")); configData.CaptchaImage.Value = captchaImage.ContentBytes; var codefield = LoginResultDocument.QuerySelector("input[name^=\"cap_code_\"]"); @@ -260,12 +260,12 @@ namespace Jackett.Common.Indexers } var searchUrl = SearchUrl + "?" + queryCollection.GetQueryString(); - var results = await RequestStringWithCookies(searchUrl); + var results = await WebRequestWithCookiesAsync(searchUrl); if (!results.ContentString.Contains("Вы зашли как:")) { // re login await ApplyConfiguration(null); - results = await RequestStringWithCookies(searchUrl); + results = await WebRequestWithCookiesAsync(searchUrl); } try { @@ -337,7 +337,7 @@ namespace Jackett.Common.Indexers public override async Task Download(Uri link) { var downloadlink = link; - var response = await RequestStringWithCookies(link.ToString()); + var response = await WebRequestWithCookiesAsync(link.ToString()); var results = response.ContentString; var SearchResultParser = new HtmlParser(); var SearchResultDocument = SearchResultParser.ParseDocument(results); diff --git a/src/Jackett.Common/Indexers/PreToMe.cs b/src/Jackett.Common/Indexers/PreToMe.cs index e2d793802..297a5d97b 100644 --- a/src/Jackett.Common/Indexers/PreToMe.cs +++ b/src/Jackett.Common/Indexers/PreToMe.cs @@ -123,7 +123,7 @@ namespace Jackett.Common.Indexers { LoadValuesFromJson(configJson); - var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); + var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); var pairs = new Dictionary { { "returnto", "%2F" }, @@ -134,7 +134,7 @@ namespace Jackett.Common.Indexers }; // Send Post - var result = await PostDataWithCookies(LoginUrl, pairs, loginPage.Cookies); + var result = await WebRequestWithCookiesAsync(LoginUrl, loginPage.Cookies, RequestType.POST, data: pairs); if (result.RedirectingTo == null) throw new ExceptionWithConfigData("Login failed. Did you use the PIN number that pretome emailed you?", configData); @@ -200,12 +200,12 @@ namespace Jackett.Common.Indexers } var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var response = await RequestStringWithCookiesAndRetry(searchUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl); if (response.IsRedirect) // re-login { await ApplyConfiguration(null); - response = await RequestStringWithCookiesAndRetry(searchUrl); + response = await RequestWithCookiesAndRetryAsync(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/RarBG.cs b/src/Jackett.Common/Indexers/RarBG.cs index a9c7c3def..6df85a31d 100644 --- a/src/Jackett.Common/Indexers/RarBG.cs +++ b/src/Jackett.Common/Indexers/RarBG.cs @@ -11,6 +11,7 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using static Jackett.Common.Models.IndexerConfig.ConfigurationData; @@ -120,7 +121,7 @@ namespace Jackett.Common.Indexers // check the token and renewal if necessary await RenewalTokenAsync(); - var response = await RequestStringWithCookiesAndRetry(BuildSearchUrl(query)); + var response = await RequestWithCookiesAndRetryAsync(BuildSearchUrl(query)); var jsonContent = JObject.Parse(response.ContentString); var errorCode = jsonContent.Value("error_code"); switch (errorCode) @@ -130,7 +131,7 @@ namespace Jackett.Common.Indexers case 2: case 4: // invalid token await RenewalTokenAsync(true); // force renewal token - response = await RequestStringWithCookiesAndRetry(BuildSearchUrl(query)); + response = await RequestWithCookiesAndRetryAsync(BuildSearchUrl(query)); jsonContent = JObject.Parse(response.ContentString); break; case 10: // imdb not found, see issue #1486 @@ -263,7 +264,7 @@ namespace Jackett.Common.Indexers { "app_id", _appId } }; var tokenUrl = ApiEndpoint + "?" + qc.GetQueryString(); - var result = await RequestStringWithCookiesAndRetry(tokenUrl); + var result = await RequestWithCookiesAndRetryAsync(tokenUrl); var json = JObject.Parse(result.ContentString); _token = json.Value("token"); _lastTokenFetch = DateTime.Now; diff --git a/src/Jackett.Common/Indexers/RevolutionTT.cs b/src/Jackett.Common/Indexers/RevolutionTT.cs index ae51113fb..8f6799c50 100644 --- a/src/Jackett.Common/Indexers/RevolutionTT.cs +++ b/src/Jackett.Common/Indexers/RevolutionTT.cs @@ -13,6 +13,7 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -207,7 +208,8 @@ namespace Jackett.Common.Indexers var rssParams = new Dictionary { { "feed", "dl" } }; - var rssPage = await PostDataWithCookies(GetRSSKeyUrl, rssParams, result.Cookies); + var rssPage = await WebRequestWithCookiesAsync( + GetRSSKeyUrl, result.Cookies, RequestType.POST, data: rssParams); var match = Regex.Match(rssPage.ContentString, "(?<=passkey\\=)([a-zA-z0-9]*)"); configData.RSSKey.Value = match.Success ? match.Value : string.Empty; if (string.IsNullOrWhiteSpace(configData.RSSKey.Value)) @@ -232,7 +234,7 @@ namespace Jackett.Common.Indexers // If query is empty, use the RSS Feed if (string.IsNullOrWhiteSpace(searchString)) { - var rssPage = await RequestStringWithCookiesAndRetry(RSSUrl + configData.RSSKey.Value); + var rssPage = await RequestWithCookiesAndRetryAsync(RSSUrl + configData.RSSKey.Value); var rssDoc = XDocument.Parse(rssPage.ContentString); foreach (var item in rssDoc.Descendants("item")) @@ -305,12 +307,12 @@ namespace Jackett.Common.Indexers } } - var results = await RequestStringWithCookiesAndRetry(searchUrl); + var results = await RequestWithCookiesAndRetryAsync(searchUrl); if (results.IsRedirect) { // re-login await ApplyConfiguration(null); - results = await RequestStringWithCookiesAndRetry(searchUrl); + results = await RequestWithCookiesAndRetryAsync(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/RuTracker.cs b/src/Jackett.Common/Indexers/RuTracker.cs index d63cea89c..e9ba74c43 100644 --- a/src/Jackett.Common/Indexers/RuTracker.cs +++ b/src/Jackett.Common/Indexers/RuTracker.cs @@ -1510,13 +1510,13 @@ namespace Jackett.Common.Indexers try { configData.CookieHeader.Value = null; - var response = await RequestStringWithCookies(LoginUrl); + var response = await WebRequestWithCookiesAsync(LoginUrl); var parser = new HtmlParser(); var doc = parser.ParseDocument(response.ContentString); var captchaimg = doc.QuerySelector("img[src^=\"https://static.t-ru.org/captcha/\"]"); if (captchaimg != null) { - var captchaImage = await RequestBytesWithCookies(captchaimg.GetAttribute("src")); + var captchaImage = await WebRequestWithCookiesAsync(captchaimg.GetAttribute("src")); configData.CaptchaImage.Value = captchaImage.ContentBytes; var codefield = doc.QuerySelector("input[name^=\"cap_code_\"]"); @@ -1591,12 +1591,12 @@ namespace Jackett.Common.Indexers } var searchUrl = SearchUrl + "?" + queryCollection.GetQueryString(); - var results = await RequestStringWithCookies(searchUrl); + var results = await WebRequestWithCookiesAsync(searchUrl); if (!results.ContentString.Contains("id=\"logged-in-username\"")) { // re login await ApplyConfiguration(null); - results = await RequestStringWithCookies(searchUrl); + results = await WebRequestWithCookiesAsync(searchUrl); } try { diff --git a/src/Jackett.Common/Indexers/SceneHD.cs b/src/Jackett.Common/Indexers/SceneHD.cs index 465d6fc6f..92cf694b9 100644 --- a/src/Jackett.Common/Indexers/SceneHD.cs +++ b/src/Jackett.Common/Indexers/SceneHD.cs @@ -91,7 +91,7 @@ namespace Jackett.Common.Indexers qc.Add("categories[" + cat + "]", "1"); var searchUrl = SearchUrl + qc.GetQueryString(); - var response = await RequestStringWithCookiesAndRetry(searchUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl); if (response.ContentString?.Contains("User not found or passkey not set") == true) throw new Exception("The passkey is invalid. Check the indexer configuration."); diff --git a/src/Jackett.Common/Indexers/SceneTime.cs b/src/Jackett.Common/Indexers/SceneTime.cs index 77c6a7476..382c31655 100644 --- a/src/Jackett.Common/Indexers/SceneTime.cs +++ b/src/Jackett.Common/Indexers/SceneTime.cs @@ -107,7 +107,7 @@ namespace Jackett.Common.Indexers WebResult loginPage; try { - loginPage = await RequestStringWithCookies(StartPageUrl, string.Empty); + loginPage = await WebRequestWithCookiesAsync(StartPageUrl, string.Empty); } catch (Exception) { @@ -198,7 +198,7 @@ namespace Jackett.Common.Indexers qParams.Add("freeleech", "on"); var searchUrl = SearchUrl + "?" + qParams.GetQueryString(); - var results = await RequestStringWithCookies(searchUrl); + var results = await WebRequestWithCookiesAsync(searchUrl); // response without results (the message is misleading) if (results.ContentString?.Contains("slow down geek!!!") == true) diff --git a/src/Jackett.Common/Indexers/Shazbat.cs b/src/Jackett.Common/Indexers/Shazbat.cs index d2ba1a25e..c7336d801 100644 --- a/src/Jackett.Common/Indexers/Shazbat.cs +++ b/src/Jackett.Common/Indexers/Shazbat.cs @@ -67,7 +67,7 @@ namespace Jackett.Common.Indexers var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, null, LoginUrl); await ConfigureIfOK(result.Cookies, result.ContentString?.Contains("glyphicon-log-out") == true, () => throw new ExceptionWithConfigData("The username and password entered do not match.", configData)); - var rssProfile = await RequestStringWithCookiesAndRetry(RSSProfile); + var rssProfile = await RequestWithCookiesAndRetryAsync(RSSProfile); var parser = new HtmlParser(); var rssDom = parser.ParseDocument(rssProfile.ContentString); configData.RSSKey.Value = rssDom.QuerySelector(".col-sm-9:nth-of-type(1)").TextContent.Trim(); @@ -89,7 +89,8 @@ namespace Jackett.Common.Indexers { {"search", query.SanitizedSearchTerm} }; - results = await PostDataWithCookiesAndRetry(SearchUrl, pairs, null, TorrentsUrl); + results = await RequestWithCookiesAndRetryAsync( + SearchUrl, null, RequestType.POST, TorrentsUrl, pairs); results = await ReloginIfNecessary(results); var parser = new HtmlParser(); var dom = parser.ParseDocument(results.ContentString); @@ -107,7 +108,7 @@ namespace Jackett.Common.Indexers { foreach (var searchUrl in searchUrls) { - results = await RequestStringWithCookies(searchUrl); + results = await WebRequestWithCookiesAsync(searchUrl); results = await ReloginIfNecessary(results); var parser = new HtmlParser(); var dom = parser.ParseDocument(results.ContentString); @@ -177,7 +178,7 @@ namespace Jackett.Common.Indexers await ApplyConfiguration(null); response.Request.Cookies = CookieHeader; - return await webclient.GetString(response.Request); + return await webclient.GetResultAsync(response.Request); } } } diff --git a/src/Jackett.Common/Indexers/ShowRSS.cs b/src/Jackett.Common/Indexers/ShowRSS.cs index 15fac9e92..c9fa9d805 100644 --- a/src/Jackett.Common/Indexers/ShowRSS.cs +++ b/src/Jackett.Common/Indexers/ShowRSS.cs @@ -60,7 +60,7 @@ namespace Jackett.Common.Indexers { var releases = new List(); var episodeSearchUrl = string.Format(SearchAllUrl); - var result = await RequestStringWithCookiesAndRetry(episodeSearchUrl); + var result = await RequestWithCookiesAndRetryAsync(episodeSearchUrl); var xmlDoc = new XmlDocument(); try diff --git a/src/Jackett.Common/Indexers/SolidTorrents.cs b/src/Jackett.Common/Indexers/SolidTorrents.cs index ae99b0cbe..868dd3725 100644 --- a/src/Jackett.Common/Indexers/SolidTorrents.cs +++ b/src/Jackett.Common/Indexers/SolidTorrents.cs @@ -103,7 +103,7 @@ namespace Jackett.Common.Indexers {"fuv", "no"} }; var fullSearchUrl = SearchUrl + "?" + queryCollection.GetQueryString(); - var result = await RequestStringWithCookies(fullSearchUrl, null, null, APIHeaders); + var result = await WebRequestWithCookiesAsync(fullSearchUrl, headers: APIHeaders); return CheckResponse(result); } diff --git a/src/Jackett.Common/Indexers/SpeedCD.cs b/src/Jackett.Common/Indexers/SpeedCD.cs index 30fda5b9e..30889a2ea 100644 --- a/src/Jackett.Common/Indexers/SpeedCD.cs +++ b/src/Jackett.Common/Indexers/SpeedCD.cs @@ -151,11 +151,11 @@ namespace Jackett.Common.Indexers } var searchUrl = SearchUrl + string.Join("/", qc); - var response = await RequestStringWithCookiesAndRetry(searchUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl); if (!response.ContentString.Contains("/logout.php")) // re-login { await DoLogin(); - response = await RequestStringWithCookiesAndRetry(searchUrl); + response = await RequestWithCookiesAndRetryAsync(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/SuperBits.cs b/src/Jackett.Common/Indexers/SuperBits.cs index b329eb83a..504b09753 100644 --- a/src/Jackett.Common/Indexers/SuperBits.cs +++ b/src/Jackett.Common/Indexers/SuperBits.cs @@ -128,7 +128,7 @@ namespace Jackett.Common.Indexers searchUrl += "?" + queryCollection.GetQueryString(); foreach (var cat in MapTorznabCapsToTrackers(query)) searchUrl += "&categories[]=" + cat; - var results = await RequestStringWithCookies(searchUrl, null, SiteLink); + var results = await WebRequestWithCookiesAsync(searchUrl, referer: SiteLink); try { diff --git a/src/Jackett.Common/Indexers/TVStore.cs b/src/Jackett.Common/Indexers/TVStore.cs index 494e826cb..ce285b28c 100644 --- a/src/Jackett.Common/Indexers/TVStore.cs +++ b/src/Jackett.Common/Indexers/TVStore.cs @@ -66,7 +66,7 @@ namespace Jackett.Common.Indexers public override async Task ApplyConfiguration(JToken configJson) { LoadValuesFromJson(configJson); - var loginPage = await RequestStringWithCookies(LoginPageUrl, string.Empty); + var loginPage = await WebRequestWithCookiesAsync(LoginPageUrl, string.Empty); var pairs = new Dictionary { {"username", configData.Username.Value}, @@ -163,7 +163,7 @@ namespace Jackett.Common.Indexers queryParams["id"] = torrentId; queryParams["now"] = DateTimeUtil.DateTimeToUnixTimestamp(DateTime.UtcNow) .ToString(CultureInfo.InvariantCulture); - var filesList = (await RequestStringWithCookiesAndRetry(SearchUrl + "?" + queryParams.GetQueryString())) + var filesList = (await RequestWithCookiesAndRetryAsync(SearchUrl + "?" + queryParams.GetQueryString())) .ContentString; var firstFileName = filesList.Split( new[] @@ -215,7 +215,7 @@ namespace Jackett.Common.Indexers /// private async Task PopulateImdbMapAsync() { - var result = await RequestStringWithCookiesAndRetry(BrowseUrl); + var result = await RequestWithCookiesAndRetryAsync(BrowseUrl); foreach (Match match in _seriesInfoMatch.Matches(result.ContentString)) { var internalId = int.Parse(match.Groups["seriesID"].Value); @@ -280,7 +280,7 @@ namespace Jackett.Common.Indexers queryParams.Add("e", query.Episode); } - var results = await RequestStringWithCookiesAndRetry(SearchUrl + "?" + queryParams.GetQueryString()); + var results = await RequestWithCookiesAndRetryAsync(SearchUrl + "?" + queryParams.GetQueryString()); // Parse page Information from result var content = results.ContentString; var splits = content.Split('\\'); @@ -302,7 +302,7 @@ namespace Jackett.Common.Indexers for (var page = startPage; page <= pages && releases.Count < query.Limit; page++) { queryParams["page"] = page.ToString(); - results = await RequestStringWithCookiesAndRetry(SearchUrl + "?" + queryParams.GetQueryString()); + results = await RequestWithCookiesAndRetryAsync(SearchUrl + "?" + queryParams.GetQueryString()); releases.AddRange(await ParseTorrentsAsync(results, releases.Count, query.Limit, previouslyParsedOnPage)); previouslyParsedOnPage = 0; } diff --git a/src/Jackett.Common/Indexers/TVVault.cs b/src/Jackett.Common/Indexers/TVVault.cs index f390713d4..2df4a7d9f 100644 --- a/src/Jackett.Common/Indexers/TVVault.cs +++ b/src/Jackett.Common/Indexers/TVVault.cs @@ -94,7 +94,7 @@ namespace Jackett.Common.Indexers qc.Add("searchstr", StripSearchString(query.GetQueryString())); var searchUrl = BrowseUrl + "?" + qc.GetQueryString(); - var results = await RequestStringWithCookies(searchUrl); + var results = await WebRequestWithCookiesAsync(searchUrl); try { var seasonRegEx = new Regex(@$"Season\s+0*{query.Season}[^\d]", RegexOptions.IgnoreCase); diff --git a/src/Jackett.Common/Indexers/Toloka.cs b/src/Jackett.Common/Indexers/Toloka.cs index 3c1a05ce4..a183f3e6a 100644 --- a/src/Jackett.Common/Indexers/Toloka.cs +++ b/src/Jackett.Common/Indexers/Toloka.cs @@ -232,12 +232,12 @@ namespace Jackett.Common.Indexers } var searchUrl = SearchUrl + "?" + queryCollection.GetQueryString(); - var results = await RequestStringWithCookies(searchUrl); + var results = await WebRequestWithCookiesAsync(searchUrl); if (!results.ContentString.Contains("logout=true")) { // re login await ApplyConfiguration(null); - results = await RequestStringWithCookies(searchUrl); + results = await WebRequestWithCookiesAsync(searchUrl); } try { diff --git a/src/Jackett.Common/Indexers/TorrenTech.cs b/src/Jackett.Common/Indexers/TorrenTech.cs index c66d55d1c..3bc2f7972 100644 --- a/src/Jackett.Common/Indexers/TorrenTech.cs +++ b/src/Jackett.Common/Indexers/TorrenTech.cs @@ -100,10 +100,10 @@ namespace Jackett.Common.Indexers } var searchUrl = IndexUrl + "?" + queryCollection.GetQueryString(); - results = await RequestStringWithCookies(searchUrl); + results = await WebRequestWithCookiesAsync(searchUrl); if (results.IsRedirect && results.RedirectingTo.Contains("CODE=show")) { - results = await RequestStringWithCookies(results.RedirectingTo); + results = await WebRequestWithCookiesAsync(results.RedirectingTo); } try { @@ -203,7 +203,7 @@ namespace Jackett.Common.Indexers public override async Task Download(Uri link) { - var response = await RequestStringWithCookies(link.ToString()); + var response = await WebRequestWithCookiesAsync(link.ToString()); var results = response.ContentString; var SearchResultParser = new HtmlParser(); var SearchResultDocument = SearchResultParser.ParseDocument(results); diff --git a/src/Jackett.Common/Indexers/TorrentBytes.cs b/src/Jackett.Common/Indexers/TorrentBytes.cs index 0f35363ab..f826ada7a 100644 --- a/src/Jackett.Common/Indexers/TorrentBytes.cs +++ b/src/Jackett.Common/Indexers/TorrentBytes.cs @@ -89,7 +89,7 @@ namespace Jackett.Common.Indexers {"returnto", "/"}, {"login", "Log in!"} }; - var loginPage = await RequestStringWithCookies(SiteLink, string.Empty); + var loginPage = await WebRequestWithCookiesAsync(SiteLink, string.Empty); var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, SiteLink, SiteLink); await ConfigureIfOK( result.Cookies, result.ContentString?.Contains("my.php") == true, () => @@ -127,12 +127,12 @@ namespace Jackett.Common.Indexers qc.Add("c" + cat, "1"); var searchUrl = SearchUrl + "?" + qc.GetQueryString(); - var response = await RequestStringWithCookiesAndRetry(searchUrl, referer: SearchUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl); if (response.IsRedirect) // re-login { await ApplyConfiguration(null); - response = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrl); + response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: SearchUrl); } try diff --git a/src/Jackett.Common/Indexers/TorrentDay.cs b/src/Jackett.Common/Indexers/TorrentDay.cs index 435f724b1..66b8fc39b 100644 --- a/src/Jackett.Common/Indexers/TorrentDay.cs +++ b/src/Jackett.Common/Indexers/TorrentDay.cs @@ -155,7 +155,7 @@ namespace Jackett.Common.Indexers else searchUrl += ";q=" + WebUtilityHelpers.UrlEncode(query.GetQueryString(), Encoding); - var results = await RequestStringWithCookiesAndRetry(searchUrl); + var results = await RequestWithCookiesAndRetryAsync(searchUrl); // Check for being logged out if (results.IsRedirect) diff --git a/src/Jackett.Common/Indexers/TorrentHeaven.cs b/src/Jackett.Common/Indexers/TorrentHeaven.cs index be7f398e7..aceab8235 100644 --- a/src/Jackett.Common/Indexers/TorrentHeaven.cs +++ b/src/Jackett.Common/Indexers/TorrentHeaven.cs @@ -119,7 +119,7 @@ namespace Jackett.Common.Indexers throw new ExceptionWithConfigData(errorMessage, configData); } - var result2 = await RequestStringWithCookies(LoginCompleteUrl, result.Cookies); + var result2 = await WebRequestWithCookiesAsync(LoginCompleteUrl, result.Cookies); await ConfigureIfOK( result2.Cookies, result2.Cookies?.Contains("pass") == true, () => throw new ExceptionWithConfigData("Didn't get a user/pass cookie", configData)); @@ -128,14 +128,14 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { - var loginPage = await RequestStringWithCookies(IndexUrl, string.Empty); + var loginPage = await WebRequestWithCookiesAsync(IndexUrl, string.Empty); var parser = new HtmlParser(); var dom = parser.ParseDocument(loginPage.ContentString); var qCaptchaImg = dom.QuerySelector("td.tablea > img"); if (qCaptchaImg != null) { var captchaUrl = SiteLink + qCaptchaImg.GetAttribute("src"); - var captchaImage = await RequestBytesWithCookies(captchaUrl, loginPage.Cookies); + var captchaImage = await WebRequestWithCookiesAsync(captchaUrl, loginPage.Cookies); configData.CaptchaImage.Value = captchaImage.ContentBytes; } else @@ -182,7 +182,7 @@ namespace Jackett.Common.Indexers foreach (var cat in MapTorznabCapsToTrackers(query)) queryCollection.Add("dirs" + cat, "1"); searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestStringWithCookies(searchUrl); + var response = await WebRequestWithCookiesAsync(searchUrl); var titleRegexp = new Regex(@"^return buildTable\('(.*?)',\s+"); try { diff --git a/src/Jackett.Common/Indexers/TorrentLeech.cs b/src/Jackett.Common/Indexers/TorrentLeech.cs index 5d8bd7de8..e5bc21bd7 100644 --- a/src/Jackett.Common/Indexers/TorrentLeech.cs +++ b/src/Jackett.Common/Indexers/TorrentLeech.cs @@ -12,6 +12,7 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; @@ -107,7 +108,7 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { - var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); + var loginPage = await WebRequestWithCookiesAsync(LoginUrl, string.Empty); var parser = new HtmlParser(); var dom = parser.ParseDocument(loginPage.ContentString); var captcha = dom.QuerySelector(".g-recaptcha"); @@ -198,12 +199,12 @@ namespace Jackett.Common.Indexers else searchUrl += "newfilter/2"; // include 0day and music - var results = await RequestStringWithCookiesAndRetry(searchUrl); + var results = await RequestWithCookiesAndRetryAsync(searchUrl); if (results.ContentString.Contains("/user/account/login")) // re-login { await DoLogin(); - results = await RequestStringWithCookiesAndRetry(searchUrl); + results = await RequestWithCookiesAndRetryAsync(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/TorrentNetwork.cs b/src/Jackett.Common/Indexers/TorrentNetwork.cs index 57bc20aae..f6e31f81d 100644 --- a/src/Jackett.Common/Indexers/TorrentNetwork.cs +++ b/src/Jackett.Common/Indexers/TorrentNetwork.cs @@ -124,7 +124,8 @@ namespace Jackett.Common.Indexers private async Task SendAPIRequest(string endpoint, object data) { var jsonData = JsonConvert.SerializeObject(data); - var result = await PostDataWithCookies(APIUrl + endpoint, null, null, SiteLink, APIHeaders, jsonData); + var result = await WebRequestWithCookiesAsync( + APIUrl + endpoint, method: RequestType.POST, referer: SiteLink, headers: APIHeaders, rawbody: jsonData); if (!result.ContentString.StartsWith("{")) // not JSON => error throw new ExceptionWithConfigData(result.ContentString, configData); dynamic json = JsonConvert.DeserializeObject(result.ContentString); diff --git a/src/Jackett.Common/Indexers/TorrentSeeds.cs b/src/Jackett.Common/Indexers/TorrentSeeds.cs index 3fea81e80..c79826b3a 100644 --- a/src/Jackett.Common/Indexers/TorrentSeeds.cs +++ b/src/Jackett.Common/Indexers/TorrentSeeds.cs @@ -12,6 +12,7 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; @@ -104,7 +105,7 @@ namespace Jackett.Common.Indexers public override async Task ApplyConfiguration(JToken configJson) { LoadValuesFromJson(configJson); - var loginPage = await RequestStringWithCookies(TokenUrl); + var loginPage = await WebRequestWithCookiesAsync(TokenUrl); var parser = new HtmlParser(); var dom = parser.ParseDocument(loginPage.ContentString); var token = dom.QuerySelector("form.form-horizontal > span"); @@ -149,12 +150,12 @@ namespace Jackett.Common.Indexers foreach (var cat in MapTorznabCapsToTrackers(query)) queryCollection.Add($"cat[{cat}]", "1"); searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestStringWithCookiesAndRetry(searchUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl); var results = response.ContentString; if (!results.Contains("/logout.php?")) { await ApplyConfiguration(null); - response = await RequestStringWithCookiesAndRetry(searchUrl); + response = await RequestWithCookiesAndRetryAsync(searchUrl); results = response.ContentString; } diff --git a/src/Jackett.Common/Indexers/TorrentSyndikat.cs b/src/Jackett.Common/Indexers/TorrentSyndikat.cs index 40e791e2c..20fc1aab4 100644 --- a/src/Jackett.Common/Indexers/TorrentSyndikat.cs +++ b/src/Jackett.Common/Indexers/TorrentSyndikat.cs @@ -111,7 +111,7 @@ namespace Jackett.Common.Indexers LoadValuesFromJson(configJson); CookieHeader = ""; - var result1 = await RequestStringWithCookies(CaptchaUrl); + var result1 = await WebRequestWithCookiesAsync(CaptchaUrl); var json1 = JObject.Parse(result1.ContentString); var captchaSelection = json1["images"][0]["hash"]; @@ -172,12 +172,12 @@ namespace Jackett.Common.Indexers var searchUrl = SearchUrl + "?" + queryCollection.GetQueryString(); - var results = await RequestStringWithCookiesAndRetry(searchUrl); + var results = await RequestWithCookiesAndRetryAsync(searchUrl); if (results.IsRedirect) { await ApplyConfiguration(null); - results = await RequestStringWithCookiesAndRetry(searchUrl); + results = await RequestWithCookiesAndRetryAsync(searchUrl); } try diff --git a/src/Jackett.Common/Indexers/TorrentsCSV.cs b/src/Jackett.Common/Indexers/TorrentsCSV.cs index 83bae844d..ec5355a9f 100644 --- a/src/Jackett.Common/Indexers/TorrentsCSV.cs +++ b/src/Jackett.Common/Indexers/TorrentsCSV.cs @@ -70,7 +70,7 @@ namespace Jackett.Common.Indexers var searchUrl = (string.IsNullOrWhiteSpace(searchString) ? NewEndpoint : SearchEndpoint) + "?" + qc.GetQueryString(); - var response = await RequestStringWithCookiesAndRetry(searchUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl); try { diff --git a/src/Jackett.Common/Indexers/XSpeeds.cs b/src/Jackett.Common/Indexers/XSpeeds.cs index 3a918d122..72341f0ff 100644 --- a/src/Jackett.Common/Indexers/XSpeeds.cs +++ b/src/Jackett.Common/Indexers/XSpeeds.cs @@ -132,14 +132,14 @@ namespace Jackett.Common.Indexers public override async Task GetConfigurationForSetup() { - var loginPage = await RequestStringWithCookies(LandingUrl); + var loginPage = await WebRequestWithCookiesAsync(LandingUrl); var parser = new HtmlParser(); var dom = parser.ParseDocument(loginPage.ContentString); var qCaptchaImg = dom.QuerySelector("img#regimage"); if (qCaptchaImg != null) { var captchaUrl = qCaptchaImg.GetAttribute("src"); - var captchaImageResponse = await RequestBytesWithCookies(captchaUrl, loginPage.Cookies, RequestType.GET, LandingUrl); + var captchaImageResponse = await WebRequestWithCookiesAsync(captchaUrl, loginPage.Cookies, RequestType.GET, LandingUrl); var captchaText = new StringItem { Name = "Captcha Text" }; var captchaImage = new ImageItem {Name = "Captcha Image", Value = captchaImageResponse.ContentBytes}; @@ -189,7 +189,8 @@ namespace Jackett.Common.Indexers {"timezone", "0"}, {"showrows", "50"} }; - var rssPage = await PostDataWithCookies(GetRSSKeyUrl, rssParams, result.Cookies); + var rssPage = await WebRequestWithCookiesAsync( + GetRSSKeyUrl, result.Cookies, RequestType.POST, data: rssParams); var match = Regex.Match(rssPage.ContentString, "(?<=secret_key\\=)([a-zA-z0-9]*)"); configData.RSSKey.Value = match.Success ? match.Value : string.Empty; if (string.IsNullOrWhiteSpace(configData.RSSKey.Value)) @@ -227,12 +228,14 @@ namespace Jackett.Common.Indexers searchParams.Add("search_type", "t_name"); } - var searchPage = await PostDataWithCookiesAndRetry(SearchUrl, searchParams, CookieHeader); + var searchPage = await RequestWithCookiesAndRetryAsync( + SearchUrl, CookieHeader, RequestType.POST, null, searchParams); // Occasionally the cookies become invalid, login again if that happens if (searchPage.IsRedirect) { await ApplyConfiguration(null); - searchPage = await PostDataWithCookiesAndRetry(SearchUrl, searchParams, CookieHeader); + searchPage = await RequestWithCookiesAndRetryAsync( + SearchUrl, CookieHeader, RequestType.POST, null, searchParams); } try diff --git a/src/Jackett.Common/Indexers/Xthor.cs b/src/Jackett.Common/Indexers/Xthor.cs index afd9e3443..adda0e449 100644 --- a/src/Jackett.Common/Indexers/Xthor.cs +++ b/src/Jackett.Common/Indexers/Xthor.cs @@ -17,6 +17,7 @@ using Jackett.Common.Utils.Clients; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; +using WebRequest = Jackett.Common.Utils.Clients.WebRequest; namespace Jackett.Common.Indexers { @@ -488,7 +489,7 @@ namespace Jackett.Common.Indexers }; // Request our first page - var results = await webclient.GetString(myIndexRequest); + var results = await webclient.GetResultAsync(myIndexRequest); if (results.Status == HttpStatusCode.InternalServerError) // See issue #2110 throw new Exception("Internal Server Error (" + results.ContentString + "), probably you reached the API limits, please reduce the number of queries"); diff --git a/src/Jackett.Common/Indexers/YTS.cs b/src/Jackett.Common/Indexers/YTS.cs index f4f3d8f73..ab3526079 100644 --- a/src/Jackett.Common/Indexers/YTS.cs +++ b/src/Jackett.Common/Indexers/YTS.cs @@ -99,7 +99,7 @@ namespace Jackett.Common.Indexers } var searchUrl = ApiEndpoint + "?" + queryCollection.GetQueryString(); - var response = await RequestStringWithCookiesAndRetry(searchUrl); + var response = await RequestWithCookiesAndRetryAsync(searchUrl); try { diff --git a/src/Jackett.Common/Services/ImdbResolver.cs b/src/Jackett.Common/Services/ImdbResolver.cs index 9b8b6785b..d35136c7e 100644 --- a/src/Jackett.Common/Services/ImdbResolver.cs +++ b/src/Jackett.Common/Services/ImdbResolver.cs @@ -37,7 +37,7 @@ namespace Jackett.Common.Services { Encoding = Encoding.UTF8 }; - var result = await WebClient.GetString(request); + var result = await WebClient.GetResultAsync(request); var movie = JsonConvert.DeserializeObject(result.ContentString); return movie; diff --git a/src/Jackett.Common/Services/UpdateService.cs b/src/Jackett.Common/Services/UpdateService.cs index 0d4f28d60..01cb46fcd 100644 --- a/src/Jackett.Common/Services/UpdateService.cs +++ b/src/Jackett.Common/Services/UpdateService.cs @@ -114,7 +114,7 @@ namespace Jackett.Common.Services try { - var response = await client.GetString(new WebRequest() + var response = await client.GetResultAsync(new WebRequest() { Url = "https://api.github.com/repos/Jackett/Jackett/releases", Encoding = Encoding.UTF8, @@ -258,11 +258,11 @@ namespace Jackett.Common.Services var url = targetAsset.Browser_download_url; - var data = await client.GetBytes(SetDownloadHeaders(new WebRequest() { Url = url, EmulateBrowser = true, Type = RequestType.GET })); + var data = await client.GetResultAsync(SetDownloadHeaders(new WebRequest() { Url = url, EmulateBrowser = true, Type = RequestType.GET })); while (data.IsRedirect) { - data = await client.GetBytes(new WebRequest() { Url = data.RedirectingTo, EmulateBrowser = true, Type = RequestType.GET }); + data = await client.GetResultAsync(new WebRequest() { Url = data.RedirectingTo, EmulateBrowser = true, Type = RequestType.GET }); } var tempDir = Path.Combine(Path.GetTempPath(), "JackettUpdate-" + version + "-" + DateTime.Now.Ticks); diff --git a/src/Jackett.Common/Utils/Clients/WebClient.cs b/src/Jackett.Common/Utils/Clients/WebClient.cs index 6350e7076..507eef3a4 100644 --- a/src/Jackett.Common/Utils/Clients/WebClient.cs +++ b/src/Jackett.Common/Utils/Clients/WebClient.cs @@ -186,38 +186,24 @@ namespace Jackett.Common.Utils.Clients return; } - public virtual async Task GetBytes(WebRequest request) + public virtual async Task GetResultAsync(WebRequest request) { - logger.Debug(string.Format("WebClient({0}).GetBytes(Url:{1})", ClientType, request.Url)); + logger.Debug(string.Format("WebClient({0}).GetResultAsync(Url:{1})", ClientType, request.Url)); PrepareRequest(request); await DelayRequest(request); var result = await Run(request); lastRequest = DateTime.Now; result.Request = request; - logger.Debug(string.Format("WebClient({0}): Returning {1} => {2} bytes", ClientType, result.Status, (result.IsRedirect ? result.RedirectingTo + " " : "") + (result.ContentBytes == null ? "" : result.ContentBytes.Length.ToString()))); + logger.Debug( + string.Format( + "WebClient({0}): Returning {1} => {2} bytes", ClientType, result.Status, + (result.IsRedirect ? result.RedirectingTo + " " : "") + + (result.ContentBytes == null ? "" : result.ContentBytes.Length.ToString()))); + if (result.Headers.TryGetValue("server", out var server) && server[0] == "cloudflare-nginx") + result.ContentString = BrowserUtil.DecodeCloudFlareProtectedEmailFromHTML(result.ContentString); return result; } - public virtual async Task GetString(WebRequest request) - { - logger.Debug(string.Format("WebClient({0}).GetString(Url:{1})", ClientType, request.Url)); - PrepareRequest(request); - await DelayRequest(request); - var result = await Run(request); - lastRequest = DateTime.Now; - result.Request = request; - WebResult stringResult = result; - - logger.Debug(string.Format("WebClient({0}): Returning {1} => {2}", ClientType, result.Status, (result.IsRedirect ? result.RedirectingTo + " " : "") + (stringResult.ContentString ?? ""))); - - if (stringResult.Headers.TryGetValue("server", out var server)) - { - if (server[0] == "cloudflare-nginx") - stringResult.ContentString = BrowserUtil.DecodeCloudFlareProtectedEmailFromHTML(stringResult.ContentString); - } - return stringResult; - } - #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously protected virtual async Task Run(WebRequest webRequest) => throw new NotImplementedException(); #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously diff --git a/src/Jackett.Test/TestUtil.cs b/src/Jackett.Test/TestUtil.cs index 652f8d604..9a18e7c8a 100644 --- a/src/Jackett.Test/TestUtil.cs +++ b/src/Jackett.Test/TestUtil.cs @@ -33,16 +33,11 @@ namespace Jackett.Test public static IContainer Container => testContainer; - public static void RegisterByteCall(WebRequest r, Func f) + //Currently not used in any Unit Tests + public static void RegisterRequestCallback(WebRequest r, Func f) { var client = testContainer.Resolve() as TestWebClient; - client.RegisterByteCall(r, f); - } - - public static void RegisterStringCall(WebRequest r, Func f) - { - var client = testContainer.Resolve() as TestWebClient; - client.RegisterStringCall(r, f); + client.RegisterRequestCallback(r, f); } public static string GetResource(string item) diff --git a/src/Jackett.Test/TestWebClient.cs b/src/Jackett.Test/TestWebClient.cs index b7a51abad..fe691a337 100644 --- a/src/Jackett.Test/TestWebClient.cs +++ b/src/Jackett.Test/TestWebClient.cs @@ -9,26 +9,20 @@ using NLog; namespace Jackett.Test { + + // Currently not used in any Unit tests. Leaving it for potential future testing purposes. public class TestWebClient : WebClient { - private readonly Dictionary> byteCallbacks = new Dictionary>(); - private readonly Dictionary> stringCallbacks = new Dictionary>(); + private readonly Dictionary> _requestCallbacks = new Dictionary>(); public TestWebClient(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc) - : base(p: p, - l: l, - c: c, - sc: sc) + : base(p, l, c, sc) { } - public void RegisterByteCall(WebRequest req, Func f) => byteCallbacks.Add(req, f); + public void RegisterRequestCallback(WebRequest req, Func f) => _requestCallbacks.Add(req, f); - public void RegisterStringCall(WebRequest req, Func f) => stringCallbacks.Add(req, f); - - public override Task GetBytes(WebRequest request) => Task.FromResult(byteCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request)); - - public override Task GetString(WebRequest request) => Task.FromResult(stringCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request)); + public override Task GetResultAsync(WebRequest request) => Task.FromResult(_requestCallbacks.First(r => r.Key.Equals(request)).Value.Invoke(request)); public override void Init() {