diff --git a/src/Jackett.Common/Plumbing/JackettModule.cs b/src/Jackett.Common/Plumbing/JackettModule.cs index fcc010c36..93c40a802 100644 --- a/src/Jackett.Common/Plumbing/JackettModule.cs +++ b/src/Jackett.Common/Plumbing/JackettModule.cs @@ -1,5 +1,3 @@ -using System; -using System.Reflection; using Autofac; using Jackett.Common.Indexers; using Jackett.Common.Indexers.Meta; @@ -45,19 +43,14 @@ namespace Jackett.Common.Plumbing switch (_runtimeSettings.ClientOverride) { case "httpclientnetcore": - RegisterWebClient(builder); - break; - case "httpclient2netcore": - RegisterWebClient(builder); - break; case "httpclient": RegisterWebClient(builder); break; + case "httpclient2netcore": case "httpclient2": RegisterWebClient(builder); break; default: - var usehttpclient = DetectMonoCompatabilityWithHttpClient(); RegisterWebClient(builder); break; } @@ -70,55 +63,5 @@ namespace Jackett.Common.Plumbing var configService = ctx.Resolve(); return configService.BuildServerConfig(_runtimeSettings); } - - private static bool DetectMonoCompatabilityWithHttpClient() - { - var usehttpclient = false; - try - { - var monotype = Type.GetType("Mono.Runtime"); - if (monotype != null) - { - var displayName = monotype.GetMethod("GetDisplayName", BindingFlags.NonPublic | BindingFlags.Static); - if (displayName != null) - { - var monoVersion = displayName.Invoke(null, null).ToString(); - var monoVersionO = new Version(monoVersion.Split(' ')[0]); - if ((monoVersionO.Major >= 4 && monoVersionO.Minor >= 8) || monoVersionO.Major >= 5) - { - // check if btls is supported - var monoSecurity = Assembly.Load("Mono.Security"); - var monoTlsProviderFactory = monoSecurity.GetType("Mono.Security.Interface.MonoTlsProviderFactory"); - if (monoTlsProviderFactory != null) - { - var isProviderSupported = monoTlsProviderFactory.GetMethod("IsProviderSupported"); - if (isProviderSupported != null) - { - var btlsSupported = (bool)isProviderSupported.Invoke(null, new string[] { "btls" }); - if (btlsSupported) - { - // initialize btls - var initialize = monoTlsProviderFactory.GetMethod("Initialize", new[] { typeof(string) }); - if (initialize != null) - { - initialize.Invoke(null, new string[] { "btls" }); - usehttpclient = true; - } - } - } - } - } - } - } - } - catch (Exception e) - { - Console.Out.WriteLine("Error while deciding which HttpWebClient to use: " + e); - } - - return usehttpclient; - } - - } } diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient.cs index 567a7bb22..024406695 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient.cs @@ -4,7 +4,6 @@ using System.Linq; using System.Net; using System.Net.Http; using System.Text; -using System.Text.RegularExpressions; using System.Threading.Tasks; using CloudflareSolverRe; using Jackett.Common.Helpers; diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs index 117020ac3..9eef78c2c 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs @@ -4,7 +4,6 @@ using System.Linq; using System.Net; using System.Net.Http; using System.Text; -using System.Text.RegularExpressions; using System.Threading.Tasks; using CloudflareSolverRe; using Jackett.Common.Helpers; @@ -69,6 +68,8 @@ namespace Jackett.Common.Utils.Clients public override void Init() { + ServicePointManager.DefaultConnectionLimit = 1000; + base.Init(); ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072; diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs deleted file mode 100644 index b0260d3fc..000000000 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs +++ /dev/null @@ -1,233 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using CloudflareSolverRe; -using Jackett.Common.Helpers; -using Jackett.Common.Models.Config; -using Jackett.Common.Services.Interfaces; -using NLog; - -namespace Jackett.Common.Utils.Clients -{ - // Compared to HttpWebClient this implementation will reuse the HttpClient instance (one per indexer). - // This should improve performance and avoid problems with too many open file handles. - public class HttpWebClient2NetCore : WebClient - { - private readonly CookieContainer cookies; - private ClearanceHandler clearanceHandlr; - private HttpClientHandler clientHandlr; - private HttpClient client; - - public HttpWebClient2NetCore(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc) - : base(p: p, - l: l, - c: c, - sc: sc) - { - cookies = new CookieContainer(); - CreateClient(); - } - - public void CreateClient() - { - clearanceHandlr = new ClearanceHandler(BrowserUtil.ChromeUserAgent) - { - MaxTries = 10 - }; - clientHandlr = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more. - UseCookies = true, - Proxy = webProxy, - UseProxy = (webProxy != null), - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate - }; - - // custom certificate validation handler (netcore version) - clientHandlr.ServerCertificateCustomValidationCallback = ValidateCertificate; - - clearanceHandlr.InnerHandler = clientHandlr; - client = new HttpClient(clearanceHandlr); - } - - // Called everytime the ServerConfig changes - public override void OnNext(ServerConfig value) - { - base.OnNext(value); - // recreate client if needed (can't just change the proxy attribute) - if (!ReferenceEquals(clientHandlr.Proxy, webProxy)) - { - CreateClient(); - } - } - - public override void Init() - { - ServicePointManager.DefaultConnectionLimit = 1000; - - base.Init(); - } - - protected override async Task Run(WebRequest webRequest) - { - HttpResponseMessage response = null; - var request = new HttpRequestMessage(); - request.Headers.ExpectContinue = false; - request.RequestUri = new Uri(webRequest.Url); - - //if (webRequest.EmulateBrowser == true) - // request.Headers.UserAgent.ParseAdd(BrowserUtil.ChromeUserAgent); - //else - // request.Headers.UserAgent.ParseAdd("Jackett/" + configService.GetVersion()); - - // clear cookies from cookiecontainer - var oldCookies = cookies.GetCookies(request.RequestUri); - foreach (Cookie oldCookie in oldCookies) - oldCookie.Expired = true; - - // add cookies to cookiecontainer - if (!string.IsNullOrWhiteSpace(webRequest.Cookies)) - { - // don't include the path, Scheme is needed for mono compatibility - var cookieUrl = new Uri(request.RequestUri.Scheme + "://" + request.RequestUri.Host); - var cookieDictionary = CookieUtil.CookieHeaderToDictionary(webRequest.Cookies); - foreach (var kv in cookieDictionary) - cookies.Add(cookieUrl, new Cookie(kv.Key, kv.Value)); - } - - if (webRequest.Headers != null) - { - foreach (var header in webRequest.Headers) - { - if (header.Key != "Content-Type") - { - request.Headers.TryAddWithoutValidation(header.Key, header.Value); - } - } - } - - if (!string.IsNullOrEmpty(webRequest.Referer)) - request.Headers.Referrer = new Uri(webRequest.Referer); - - if (!string.IsNullOrEmpty(webRequest.RawBody)) - { - var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast?>().FirstOrDefault(); - if (type.HasValue) - { - var str = new StringContent(webRequest.RawBody); - str.Headers.Remove("Content-Type"); - str.Headers.Add("Content-Type", type.Value.Value); - request.Content = str; - } - else - request.Content = new StringContent(webRequest.RawBody); - request.Method = HttpMethod.Post; - } - else if (webRequest.Type == RequestType.POST) - { - if (webRequest.PostData != null) - request.Content = FormUrlEncodedContentWithEncoding(webRequest.PostData, webRequest.Encoding); - request.Method = HttpMethod.Post; - } - else - { - request.Method = HttpMethod.Get; - } - - response = await client.SendAsync(request); - - var result = new WebClientByteResult - { - ContentBytes = await response.Content.ReadAsByteArrayAsync() - }; - - foreach (var header in response.Headers) - { - var value = header.Value; - result.Headers[header.Key.ToLowerInvariant()] = value.ToArray(); - } - - // some cloudflare clients are using a refresh header - // Pull it out manually - if (response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh")) - { - var refreshHeaders = response.Headers.GetValues("Refresh"); - var redirval = ""; - var redirtime = 0; - if (refreshHeaders != null) - { - foreach (var value in refreshHeaders) - { - var start = value.IndexOf("="); - var end = value.IndexOf(";"); - var len = value.Length; - if (start > -1) - { - redirval = value.Substring(start + 1); - result.RedirectingTo = redirval; - // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature - // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally - // it shoudln't include service unavailable..only if we have this redirect header. - response.StatusCode = System.Net.HttpStatusCode.Redirect; - redirtime = int.Parse(value.Substring(0, end)); - System.Threading.Thread.Sleep(redirtime * 1000); - } - } - } - } - if (response.Headers.Location != null) - { - result.RedirectingTo = response.Headers.Location.ToString(); - } - // Mono won't add the baseurl to relative redirects. - // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php" - // See issue #1200 - if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://")) - { - // URL decoding apparently is needed to, without it e.g. Demonoid download is broken - // TODO: is it always needed (not just for relative redirects)? - var newRedirectingTo = WebUtilityHelpers.UrlDecode(result.RedirectingTo, webRequest.Encoding); - if (newRedirectingTo.StartsWith("file:////")) // Location without protocol but with host (only add scheme) - newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + ":"); - else - newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host); - logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo); - result.RedirectingTo = newRedirectingTo; - } - result.Status = response.StatusCode; - - // Compatiblity issue between the cookie format and httpclient - // Pull it out manually ignoring the expiry date then set it manually - // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer - var responseCookies = new List>(); - - if (response.Headers.TryGetValues("set-cookie", out var cookieHeaders)) - { - foreach (var value in cookieHeaders) - { - logger.Debug(value); - var nameSplit = value.IndexOf('='); - if (nameSplit > -1) - { - responseCookies.Add(new Tuple(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";")); - } - } - - var cookieBuilder = new StringBuilder(); - foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1)) - { - cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2); - } - result.Cookies = cookieBuilder.ToString().Trim(); - } - ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); - return result; - } - } -} diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs b/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs deleted file mode 100644 index 71cfa3b6e..000000000 --- a/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs +++ /dev/null @@ -1,214 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using CloudflareSolverRe; -using Jackett.Common.Helpers; -using Jackett.Common.Models.Config; -using Jackett.Common.Services.Interfaces; -using NLog; - -namespace Jackett.Common.Utils.Clients -{ - // custom HttpWebClient based WebClient for netcore (due to changed custom certificate validation API) - public class HttpWebClientNetCore : WebClient - { - public HttpWebClientNetCore(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc) - : base(p: p, - l: l, - c: c, - sc: sc) - { - } - public override void Init() - { - ServicePointManager.DefaultConnectionLimit = 1000; - - base.Init(); - } - - protected override async Task Run(WebRequest webRequest) - { - ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072; - - var cookies = new CookieContainer(); - if (!string.IsNullOrWhiteSpace(webRequest.Cookies)) - { - // don't include the path, Scheme is needed for mono compatibility - var requestUri = new Uri(webRequest.Url); - var cookieUrl = new Uri(requestUri.Scheme + "://" + requestUri.Host); - var cookieDictionary = CookieUtil.CookieHeaderToDictionary(webRequest.Cookies); - foreach (var kv in cookieDictionary) - cookies.Add(cookieUrl, new Cookie(kv.Key, kv.Value)); - } - - var userAgent = webRequest.EmulateBrowser.Value ? BrowserUtil.ChromeUserAgent : "Jackett/" + configService.GetVersion(); - - using (var clearanceHandlr = new ClearanceHandler(userAgent)) - { - clearanceHandlr.MaxTries = 10; - using (var clientHandlr = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more. - UseCookies = true, - Proxy = webProxy, - UseProxy = (webProxy != null), - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate - }) - { - // custom certificate validation handler (netcore version) - clientHandlr.ServerCertificateCustomValidationCallback = ValidateCertificate; - - clearanceHandlr.InnerHandler = clientHandlr; - using (var client = new HttpClient(clearanceHandlr)) - { - //if (webRequest.EmulateBrowser == true) - // client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent); - //else - // client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion()); - - HttpResponseMessage response = null; - using (var request = new HttpRequestMessage()) - { - request.Headers.ExpectContinue = false; - request.RequestUri = new Uri(webRequest.Url); - - if (webRequest.Headers != null) - { - foreach (var header in webRequest.Headers) - { - if (header.Key != "Content-Type") - { - request.Headers.TryAddWithoutValidation(header.Key, header.Value); - } - } - } - - if (!string.IsNullOrEmpty(webRequest.Referer)) - request.Headers.Referrer = new Uri(webRequest.Referer); - - if (!string.IsNullOrEmpty(webRequest.RawBody)) - { - var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast?>().FirstOrDefault(); - if (type.HasValue) - { - var str = new StringContent(webRequest.RawBody); - str.Headers.Remove("Content-Type"); - str.Headers.Add("Content-Type", type.Value.Value); - request.Content = str; - } - else - request.Content = new StringContent(webRequest.RawBody); - request.Method = HttpMethod.Post; - } - else if (webRequest.Type == RequestType.POST) - { - if (webRequest.PostData != null) - request.Content = FormUrlEncodedContentWithEncoding(webRequest.PostData, webRequest.Encoding); - request.Method = HttpMethod.Post; - } - else - { - request.Method = HttpMethod.Get; - } - - using (response = await client.SendAsync(request)) - { - var result = new WebClientByteResult - { - ContentBytes = await response.Content.ReadAsByteArrayAsync() - }; - - foreach (var header in response.Headers) - { - var value = header.Value; - result.Headers[header.Key.ToLowerInvariant()] = value.ToArray(); - } - - // some cloudflare clients are using a refresh header - // Pull it out manually - if (response.StatusCode == HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh")) - { - var refreshHeaders = response.Headers.GetValues("Refresh"); - var redirval = ""; - var redirtime = 0; - if (refreshHeaders != null) - { - foreach (var value in refreshHeaders) - { - var start = value.IndexOf("="); - var end = value.IndexOf(";"); - var len = value.Length; - if (start > -1) - { - redirval = value.Substring(start + 1); - result.RedirectingTo = redirval; - // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature - // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally - // it shoudln't include service unavailable..only if we have this redirect header. - response.StatusCode = System.Net.HttpStatusCode.Redirect; - redirtime = int.Parse(value.Substring(0, end)); - System.Threading.Thread.Sleep(redirtime * 1000); - } - } - } - } - if (response.Headers.Location != null) - { - result.RedirectingTo = response.Headers.Location.ToString(); - } - // Mono won't add the baseurl to relative redirects. - // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php" - // See issue #1200 - if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://")) - { - // URL decoding apparently is needed to, without it e.g. Demonoid download is broken - // TODO: is it always needed (not just for relative redirects)? - var newRedirectingTo = WebUtilityHelpers.UrlDecode(result.RedirectingTo, webRequest.Encoding); - if (newRedirectingTo.StartsWith("file:////")) // Location without protocol but with host (only add scheme) - newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + ":"); - else - newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host); - logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo); - result.RedirectingTo = newRedirectingTo; - } - result.Status = response.StatusCode; - - // Compatiblity issue between the cookie format and httpclient - // Pull it out manually ignoring the expiry date then set it manually - // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer - var responseCookies = new List>(); - - if (response.Headers.TryGetValues("set-cookie", out var cookieHeaders)) - { - foreach (var value in cookieHeaders) - { - var nameSplit = value.IndexOf('='); - if (nameSplit > -1) - { - responseCookies.Add(new Tuple(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";")); - } - } - - var cookieBuilder = new StringBuilder(); - foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1)) - { - cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2); - } - result.Cookies = cookieBuilder.ToString().Trim(); - } - ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); - return result; - } - } - } - } - } - } - } -}