core: refactor http webclient part 7 #8529 (#7689)

Move web-result encoding logic to internal lazy evaluation

src/Jackett.Common/Utils/Clients/BaseWebResult.cs

@@ -1,22 +1,58 @@
+using System;
 using System.Collections.Generic;
 using System.Net;
 using System.Text;
+using System.Text.RegularExpressions;
 
 namespace Jackett.Common.Utils.Clients
 {
     public abstract class BaseWebResult
     {
-        public Encoding Encoding { get; set; }
+        private Encoding _encoding;
+
+        public Encoding Encoding
+        {
+            get
+            {
+                if (_encoding != null)
+                    return _encoding;
+                if (Request.Encoding != null)
+                    _encoding = Request.Encoding;
+                else if (Headers.ContainsKey("content-type"))
+                {
+                    var charsetRegexMatch = Regex.Match(Headers["content-type"][0], @"charset=([\w-]+)", RegexOptions.Compiled);
+                    if (charsetRegexMatch.Success)
+                    {
+                        var charset = charsetRegexMatch.Groups[1].Value;
+                        try
+                        {
+                            _encoding = Encoding.GetEncoding(charset);
+                        }
+                        catch (ArgumentException)
+                        {
+                            // Encoding not found or not enabled on current machine.
+                        }
+                    }
+                }
+
+                _encoding ??= Encoding.UTF8;
+
+                return _encoding;
+            }
+            set => _encoding = value;
+        }
+
         public HttpStatusCode Status { get; set; }
         public string Cookies { get; set; }
         public string RedirectingTo { get; set; }
         public WebRequest Request { get; set; }
-        public Dictionary<string, string[]> Headers = new Dictionary<string, string[]>();
+        public Dictionary<string, string[]> Headers { get; protected set; } =
+            new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase);
 
-        public bool IsRedirect => Status == System.Net.HttpStatusCode.Redirect ||
+        public bool IsRedirect => Status == HttpStatusCode.Redirect ||
-                        Status == System.Net.HttpStatusCode.RedirectKeepVerb ||
+                                  Status == HttpStatusCode.RedirectKeepVerb ||
-                        Status == System.Net.HttpStatusCode.RedirectMethod ||
+                                  Status == HttpStatusCode.RedirectMethod ||
-                        Status == System.Net.HttpStatusCode.Found ||
+                                  Status == HttpStatusCode.Found ||
-                        Status == System.Net.HttpStatusCode.MovedPermanently;
+                                  Status == HttpStatusCode.MovedPermanently;
     }
 }

src/Jackett.Common/Utils/Clients/HttpWebClient.cs

@@ -294,40 +294,6 @@ namespace Jackett.Common.Utils.Clients
                                     result.Cookies = cookieBuilder.ToString().Trim();
                                 }
                                 ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
-                                Encoding encoding = null;
-                                if (webRequest.Encoding != null)
-                                {
-                                    encoding = webRequest.Encoding;
-                                }
-                                else if (result.Headers.ContainsKey("content-type"))
-                                {
-                                    var CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled);
-                                    var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]);
-                                    if (CharsetRegexMatch.Success)
-                                    {
-                                        var charset = CharsetRegexMatch.Groups[1].Value;
-                                        try
-                                        {
-                                            encoding = Encoding.GetEncoding(charset);
-                                        }
-                                        catch (Exception ex)
-                                        {
-                                            logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Error loading encoding {2} based on header {3}: {4}", ClientType, webRequest.Url, charset, result.Headers["content-type"][0], ex));
-                                        }
-                                    }
-                                    else
-                                    {
-                                        logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Got header without charset: {2}", ClientType, webRequest.Url, result.Headers["content-type"][0]));
-                                    }
-                                }
-
-                                if (encoding == null)
-                                {
-                                    logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): No encoding detected, defaulting to UTF-8", ClientType, webRequest.Url));
-                                    encoding = Encoding.UTF8;
-                                }
-
-                                result.Encoding = encoding;
                                 return result;
                             }
                         }

src/Jackett.Common/Utils/Clients/HttpWebClient2.cs

@@ -314,40 +314,6 @@ namespace Jackett.Common.Utils.Clients
                 result.Cookies = cookieBuilder.ToString().Trim();
             }
             ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
-            Encoding encoding = null;
-            if (webRequest.Encoding != null)
-            {
-                encoding = webRequest.Encoding;
-            }
-            else if (result.Headers.ContainsKey("content-type"))
-            {
-                var CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled);
-                var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]);
-                if (CharsetRegexMatch.Success)
-                {
-                    var charset = CharsetRegexMatch.Groups[1].Value;
-                    try
-                    {
-                        encoding = Encoding.GetEncoding(charset);
-                    }
-                    catch (Exception ex)
-                    {
-                        logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Error loading encoding {2} based on header {3}: {4}", ClientType, webRequest.Url, charset, result.Headers["content-type"][0], ex));
-                    }
-                }
-                else
-                {
-                    logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Got header without charset: {2}", ClientType, webRequest.Url, result.Headers["content-type"][0]));
-                }
-            }
-
-            if (encoding == null)
-            {
-                logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): No encoding detected, defaulting to UTF-8", ClientType, webRequest.Url));
-                encoding = Encoding.UTF8;
-            }
-
-            result.Encoding = encoding;
             return result;
         }
 

src/Jackett.Common/Utils/Clients/HttpWebClient2NetCore.cs

@@ -314,40 +314,6 @@ namespace Jackett.Common.Utils.Clients
                 result.Cookies = cookieBuilder.ToString().Trim();
             }
             ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
-            Encoding encoding = null;
-            if (webRequest.Encoding != null)
-            {
-                encoding = webRequest.Encoding;
-            }
-            else if (result.Headers.ContainsKey("content-type"))
-            {
-                var CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled);
-                var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]);
-                if (CharsetRegexMatch.Success)
-                {
-                    var charset = CharsetRegexMatch.Groups[1].Value;
-                    try
-                    {
-                        encoding = Encoding.GetEncoding(charset);
-                    }
-                    catch (Exception ex)
-                    {
-                        logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Error loading encoding {2} based on header {3}: {4}", ClientType, webRequest.Url, charset, result.Headers["content-type"][0], ex));
-                    }
-                }
-                else
-                {
-                    logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Got header without charset: {2}", ClientType, webRequest.Url, result.Headers["content-type"][0]));
-                }
-            }
-
-            if (encoding == null)
-            {
-                logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): No encoding detected, defaulting to UTF-8", ClientType, webRequest.Url));
-                encoding = Encoding.UTF8;
-            }
-
-            result.Encoding = encoding;
             return result;
         }
 

src/Jackett.Common/Utils/Clients/HttpWebClientNetCore.cs

@@ -297,40 +297,6 @@ namespace Jackett.Common.Utils.Clients
                                     result.Cookies = cookieBuilder.ToString().Trim();
                                 }
                                 ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
-                                Encoding encoding = null;
-                                if (webRequest.Encoding != null)
-                                {
-                                    encoding = webRequest.Encoding;
-                                }
-                                else if (result.Headers.ContainsKey("content-type"))
-                                {
-                                    var CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled);
-                                    var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]);
-                                    if (CharsetRegexMatch.Success)
-                                    {
-                                        var charset = CharsetRegexMatch.Groups[1].Value;
-                                        try
-                                        {
-                                            encoding = Encoding.GetEncoding(charset);
-                                        }
-                                        catch (Exception ex)
-                                        {
-                                            logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Error loading encoding {2} based on header {3}: {4}", ClientType, webRequest.Url, charset, result.Headers["content-type"][0], ex));
-                                        }
-                                    }
-                                    else
-                                    {
-                                        logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): Got header without charset: {2}", ClientType, webRequest.Url, result.Headers["content-type"][0]));
-                                    }
-                                }
-
-                                if (encoding == null)
-                                {
-                                    logger.Error(string.Format("WebClient({0}).GetString(Url:{1}): No encoding detected, defaulting to UTF-8", ClientType, webRequest.Url));
-                                    encoding = Encoding.UTF8;
-                                }
-
-                                result.Encoding = encoding;
                                 return result;
                             }
                         }