Feature/netcore preparation (#2072)

* Use platform detection that works on mono 4.6+

* Move to use package reference for restoring nuget packages.

* DateTimeRoutines does not have Nuget packages that support .NET Standard (and therefore .NET Core). We will have to include them for now until we can get rid of this dependency.

* Start spliting some interfaces into their own files - this will help by allowing us to split them out in the future into a seperate project so the actual implementations can stay within their respective architectures when required

* Move out common libraries

* Few more tidy up tasks to get things working with .NET Standard

* Restructure the solution layout

* Encoding work to reduce rework later on platforms without Windows codepages (or require compliance with RFC1345)

* Move folder structure around to have more natural layout of the solutions

* DI server configuration to get rid of "temporary" hack and dependency circle for serverservice

* Make all encoding consistent to match the expected encoding casing for earlier versions of mono.

src/Jackett.Common/Utils/Clients/HttpWebClient2.cs

@@ -0,0 +1,271 @@
+using AutoMapper;
+using CloudFlareUtilities;
+using Jackett.Models;
+using Jackett.Services;
+using NLog;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Net.Security;
+using System.Security.Cryptography.X509Certificates;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using Jackett.Services.Interfaces;
+using Jacket.Common;
+using Jackett.Models.Config;
+
+namespace Jackett.Utils.Clients
+{
+    // Compared to HttpWebClient this implementation will reuse the HttpClient instance (one per indexer).
+    // This should improve performance and avoid problems with too man open file handles.
+    public class HttpWebClient2 : WebClient
+    {
+        CookieContainer cookies;
+        ClearanceHandler clearanceHandlr;
+        HttpClientHandler clientHandlr;
+        HttpClient client;
+
+        static protected Dictionary<string, ICollection<string>> trustedCertificates = new Dictionary<string, ICollection<string>>();
+
+        public HttpWebClient2(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc)
+            : base(p: p,
+                   l: l,
+                   c: c,
+                   sc: sc)
+        {
+
+            cookies = new CookieContainer();
+            var useProxy = false;
+            WebProxy proxyServer = null;
+            var proxyUrl = serverConfig.ProxyUrl;
+            if (!string.IsNullOrWhiteSpace(proxyUrl))
+            {
+                if (serverConfig.ProxyPort.HasValue)
+                {
+                    proxyServer = new WebProxy(proxyUrl, serverConfig.ProxyPort.Value);
+                }
+                else
+                {
+                    proxyServer = new WebProxy(proxyUrl);
+                }
+                var username = serverConfig.ProxyUsername;
+                var password = serverConfig.ProxyPassword;
+                if (!string.IsNullOrWhiteSpace(username) && !string.IsNullOrWhiteSpace(password))
+                {
+                    var creds = new NetworkCredential(username, password);
+                    proxyServer.Credentials = creds;
+                }
+                proxyServer.BypassProxyOnLocal = false;
+                useProxy = true;
+            }
+
+            clearanceHandlr = new ClearanceHandler();
+            clientHandlr = new HttpClientHandler
+            {
+                CookieContainer = cookies,
+                AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
+                UseCookies = true,
+                Proxy = proxyServer,
+                UseProxy = useProxy,
+                AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
+            };
+
+            clearanceHandlr.InnerHandler = clientHandlr;
+            client = new HttpClient(clearanceHandlr);
+        }
+
+        override public void Init()
+        {
+            if (JackettStartup.IgnoreSslErrors == true)
+            {
+                logger.Info(string.Format("HttpWebClient2: Disabling certificate validation"));
+                ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, sslPolicyErrors) => { return true; };
+            }
+
+            ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;
+
+            // custom handler for our own internal certificates
+            ServicePointManager.ServerCertificateValidationCallback += delegate (object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors)
+            {
+                if (sender.GetType() != typeof(HttpWebRequest))
+                    return sslPolicyErrors == SslPolicyErrors.None;
+
+                var request = (HttpWebRequest)sender;
+                var hash = certificate.GetCertHashString();
+
+                ICollection<string> hosts;
+
+                trustedCertificates.TryGetValue(hash, out hosts);
+                if (hosts != null)
+                {
+                    if (hosts.Contains(request.Host))
+                        return true;
+                }
+                return sslPolicyErrors == SslPolicyErrors.None;
+            };
+        }
+
+        override protected async Task<WebClientByteResult> Run(WebRequest webRequest)
+        {
+            HttpResponseMessage response = null;
+            var request = new HttpRequestMessage();
+            request.Headers.ExpectContinue = false;
+            request.RequestUri = new Uri(webRequest.Url);
+
+            if (webRequest.EmulateBrowser == true)
+                request.Headers.UserAgent.ParseAdd(BrowserUtil.ChromeUserAgent);
+            else
+                request.Headers.UserAgent.ParseAdd("Jackett/" + configService.GetVersion());
+
+            // clear cookies from cookiecontainer
+            var oldCookies = cookies.GetCookies(request.RequestUri);
+            foreach (Cookie oldCookie in oldCookies)
+            {
+                oldCookie.Expired = true;
+            }
+
+            if (!string.IsNullOrEmpty(webRequest.Cookies))
+            {
+                // add cookies to cookiecontainer
+                var cookieUrl = new Uri(request.RequestUri.Scheme + "://" + request.RequestUri.Host); // don't include the path, Scheme is needed for mono compatibility
+                foreach (var ccookiestr in webRequest.Cookies.Split(';'))
+                {
+                    var cookiestrparts = ccookiestr.Split('=');
+                    var name = cookiestrparts[0].Trim();
+                    if (string.IsNullOrWhiteSpace(name))
+                        continue;
+                    var value = "";
+                    if (cookiestrparts.Length >= 2)
+                        value = cookiestrparts[1].Trim();
+                    var cookie = new Cookie(name, value);
+                    cookies.Add(cookieUrl, cookie);
+                }
+            }
+
+            if (webRequest.Headers != null)
+            {
+                foreach (var header in webRequest.Headers)
+                {
+                    if (header.Key != "Content-Type")
+                    {
+                        request.Headers.TryAddWithoutValidation(header.Key, header.Value);
+                    }
+                }
+            }
+
+            if (!string.IsNullOrEmpty(webRequest.Referer))
+                request.Headers.Referrer = new Uri(webRequest.Referer);
+
+            if (!string.IsNullOrEmpty(webRequest.RawBody))
+            {
+                var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast<KeyValuePair<string,string>?>().FirstOrDefault();
+                if (type.HasValue)
+                {
+                    var str = new StringContent(webRequest.RawBody);
+                    str.Headers.Remove("Content-Type");
+                    str.Headers.Add("Content-Type", type.Value.Value);
+                    request.Content = str;
+                }
+                else
+                    request.Content = new StringContent(webRequest.RawBody);
+                request.Method = HttpMethod.Post;
+            }
+            else if (webRequest.Type == RequestType.POST)
+            {
+                if (webRequest.PostData != null)
+                    request.Content = new FormUrlEncodedContent(webRequest.PostData);
+                request.Method = HttpMethod.Post;
+            }
+            else
+            {
+                request.Method = HttpMethod.Get;
+            }
+
+            response = await client.SendAsync(request);
+
+            var result = new WebClientByteResult();
+            result.Content = await response.Content.ReadAsByteArrayAsync();
+
+            foreach (var header in response.Headers)
+            {
+                IEnumerable<string> value = header.Value;
+                result.Headers[header.Key.ToLowerInvariant()] = value.ToArray();
+            }
+
+            // some cloudflare clients are using a refresh header
+            // Pull it out manually 
+            if (response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
+            {
+                var refreshHeaders = response.Headers.GetValues("Refresh");
+                var redirval = "";
+                var redirtime = 0;
+                if (refreshHeaders != null)
+                {
+                    foreach (var value in refreshHeaders)
+                    {
+                        var start = value.IndexOf("=");
+                        var end = value.IndexOf(";");
+                        var len = value.Length;
+                        if (start > -1)
+                        {
+                            redirval = value.Substring(start + 1);
+                            result.RedirectingTo = redirval;
+                            // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
+                            // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
+                            // it shoudln't include service unavailable..only if we have this redirect header.
+                            response.StatusCode = System.Net.HttpStatusCode.Redirect;
+                            redirtime = Int32.Parse(value.Substring(0, end));
+                            System.Threading.Thread.Sleep(redirtime * 1000);
+                        }
+                    }
+                }
+            }
+            if (response.Headers.Location != null)
+            {
+                result.RedirectingTo = response.Headers.Location.ToString();
+            }
+            // Mono won't add the baseurl to relative redirects.
+            // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
+            // See issue #1200
+            if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://"))
+            {
+                var newRedirectingTo = result.RedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host);
+                logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo);
+                result.RedirectingTo = newRedirectingTo;
+            }
+            result.Status = response.StatusCode;
+
+            // Compatiblity issue between the cookie format and httpclient
+            // Pull it out manually ignoring the expiry date then set it manually
+            // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
+            IEnumerable<string> cookieHeaders;
+            var responseCookies = new List<Tuple<string, string>>();
+
+            if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
+            {
+                foreach (var value in cookieHeaders)
+                {
+                    logger.Debug(value);
+                    var nameSplit = value.IndexOf('=');
+                    if (nameSplit > -1)
+                    {
+                        responseCookies.Add(new Tuple<string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';')))+";"));
+                    }
+                }
+
+                var cookieBuilder = new StringBuilder();
+                foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
+                {
+                    cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
+                }
+                result.Cookies = cookieBuilder.ToString().Trim();
+            }
+            ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
+            return result;
+        }
+    }
+}