From 79fa3b9e0d7a8c43a3684e3f2cd2a871c8c2fc76 Mon Sep 17 00:00:00 2001 From: Cory Date: Mon, 16 Mar 2020 15:18:15 -0500 Subject: [PATCH] core: refactor http webclient part 7 #8529 (#7689) Move web-result encoding logic to internal lazy evaluation --- .../Utils/Clients/BaseWebResult.cs | 50 ++++++++++++++++--- .../Utils/Clients/HttpWebClient.cs | 34 ------------- .../Utils/Clients/HttpWebClient2.cs | 34 ------------- .../Utils/Clients/HttpWebClient2NetCore.cs | 34 ------------- .../Utils/Clients/HttpWebClientNetCore.cs | 34 ------------- 5 files changed, 43 insertions(+), 143 deletions(-) diff --git a/src/Jackett.Common/Utils/Clients/BaseWebResult.cs b/src/Jackett.Common/Utils/Clients/BaseWebResult.cs index cf47df862..ed171261b 100644 --- a/src/Jackett.Common/Utils/Clients/BaseWebResult.cs +++ b/src/Jackett.Common/Utils/Clients/BaseWebResult.cs @@ -1,22 +1,58 @@ +using System; using System.Collections.Generic; using System.Net; using System.Text; +using System.Text.RegularExpressions; namespace Jackett.Common.Utils.Clients { public abstract class BaseWebResult { - public Encoding Encoding { get; set; } + private Encoding _encoding; + + public Encoding Encoding + { + get + { + if (_encoding != null) + return _encoding; + if (Request.Encoding != null) + _encoding = Request.Encoding; + else if (Headers.ContainsKey("content-type")) + { + var charsetRegexMatch = Regex.Match(Headers["content-type"][0], @"charset=([\w-]+)", RegexOptions.Compiled); + if (charsetRegexMatch.Success) + { + var charset = charsetRegexMatch.Groups[1].Value; + try + { + _encoding = Encoding.GetEncoding(charset); + } + catch (ArgumentException) + { + // Encoding not found or not enabled on current machine. + } + } + } + + _encoding ??= Encoding.UTF8; + + return _encoding; + } + set => _encoding = value; + } + public HttpStatusCode Status { get; set; } public string Cookies { get; set; } public string RedirectingTo { get; set; } public WebRequest Request { get; set; } - public Dictionary Headers = new Dictionary(); + public Dictionary Headers { get; protected set; } = + new Dictionary(StringComparer.OrdinalIgnoreCase); - public bool IsRedirect => Status == System.Net.HttpStatusCode.Redirect || - Status == System.Net.HttpStatusCode.RedirectKeepVerb || - Status == System.Net.HttpStatusCode.RedirectMethod || - Status == System.Net.HttpStatusCode.Found || - Status == System.Net.HttpStatusCode.MovedPermanently; + public bool IsRedirect => Status == HttpStatusCode.Redirect || + Status == HttpStatusCode.RedirectKeepVerb || + Status == HttpStatusCode.RedirectMethod || + Status == HttpStatusCode.Found || + Status == HttpStatusCode.MovedPermanently; } } diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient.cs index 17b15029c..184adce0f 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient.cs @@ -294,40 +294,6 @@ namespace Jackett.Common.Utils.Clients result.Cookies = cookieBuilder.ToString().Trim(); } ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); - Encoding encoding = null; - if (webRequest.Encoding != null) - { - encoding = webRequest.Encoding; - } - else if (result.Headers.ContainsKey("content-type")) - { - var CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled); - var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]); - if (CharsetRegexMatch.Success) - { - var charset = CharsetRegexMatch.Groups[1].Value; - try - { - encoding = Encoding.GetEncoding(charset); - } - catch (Exception ex) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Error loading encoding {2} based on header {3}: {4}", ClientType, webRequest.Url, charset, result.Headers["content-type"][0], ex)); - } - } - else - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Got header without charset: {2}", ClientType, webRequest.Url, result.Headers["content-type"][0])); - } - } - - if (encoding == null) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): No encoding detected, defaulting to UTF-8", ClientType, webRequest.Url)); - encoding = Encoding.UTF8; - } - - result.Encoding = encoding; return result; } } diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs index 5aa26bc04..f8ea2f9d8 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient2.cs @@ -314,40 +314,6 @@ namespace Jackett.Common.Utils.Clients result.Cookies = cookieBuilder.ToString().Trim(); } ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); - Encoding encoding = null; - if (webRequest.Encoding != null) - { - encoding = webRequest.Encoding; - } - else if (result.Headers.ContainsKey("content-type")) - { - var CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled); - var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]); - if (CharsetRegexMatch.Success) - { - var charset = CharsetRegexMatch.Groups[1].Value; - try - { - encoding = Encoding.GetEncoding(charset); - } - catch (Exception ex) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Error loading encoding {2} based on header {3}: {4}", ClientType, webRequest.Url, charset, result.Headers["content-type"][0], ex)); - } - } - else - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Got header without charset: {2}", ClientType, webRequest.Url, result.Headers["content-type"][0])); - } - } - - if (encoding == null) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): No encoding detected, defaulting to UTF-8", ClientType, webRequest.Url)); - encoding = Encoding.UTF8; - } - - result.Encoding = encoding; return result; } diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs b/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs index 131924b64..34f111bb5 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs @@ -314,40 +314,6 @@ namespace Jackett.Common.Utils.Clients result.Cookies = cookieBuilder.ToString().Trim(); } ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); - Encoding encoding = null; - if (webRequest.Encoding != null) - { - encoding = webRequest.Encoding; - } - else if (result.Headers.ContainsKey("content-type")) - { - var CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled); - var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]); - if (CharsetRegexMatch.Success) - { - var charset = CharsetRegexMatch.Groups[1].Value; - try - { - encoding = Encoding.GetEncoding(charset); - } - catch (Exception ex) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Error loading encoding {2} based on header {3}: {4}", ClientType, webRequest.Url, charset, result.Headers["content-type"][0], ex)); - } - } - else - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Got header without charset: {2}", ClientType, webRequest.Url, result.Headers["content-type"][0])); - } - } - - if (encoding == null) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): No encoding detected, defaulting to UTF-8", ClientType, webRequest.Url)); - encoding = Encoding.UTF8; - } - - result.Encoding = encoding; return result; } diff --git a/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs b/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs index 58411a364..afa458ee5 100644 --- a/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs +++ b/src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs @@ -297,40 +297,6 @@ namespace Jackett.Common.Utils.Clients result.Cookies = cookieBuilder.ToString().Trim(); } ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); - Encoding encoding = null; - if (webRequest.Encoding != null) - { - encoding = webRequest.Encoding; - } - else if (result.Headers.ContainsKey("content-type")) - { - var CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled); - var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]); - if (CharsetRegexMatch.Success) - { - var charset = CharsetRegexMatch.Groups[1].Value; - try - { - encoding = Encoding.GetEncoding(charset); - } - catch (Exception ex) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Error loading encoding {2} based on header {3}: {4}", ClientType, webRequest.Url, charset, result.Headers["content-type"][0], ex)); - } - } - else - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Got header without charset: {2}", ClientType, webRequest.Url, result.Headers["content-type"][0])); - } - } - - if (encoding == null) - { - logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): No encoding detected, defaulting to UTF-8", ClientType, webRequest.Url)); - encoding = Encoding.UTF8; - } - - result.Encoding = encoding; return result; } }