Make lib curl more stable

src/Jackett/CurlHelper.cs

@@ -12,8 +12,16 @@ using System.Net;
 
 namespace Jackett
 {
-    public static class CurlHelper
+    public class CurlHelper
     {
+        private static readonly object instance = new object();
+
+        static CurlHelper()
+        {
+            Engine.Logger.Debug("LibCurl init" + Curl.GlobalInit(CurlInitFlag.All).ToString());
+            Engine.Logger.Debug("LibCurl version " + Curl.Version);
+        }
+
         public class CurlRequest
         {
 
@@ -39,17 +47,11 @@ namespace Jackett
         public class CurlResponse
         {
             public Dictionary<string, string> Headers { get; private set; }
-
             public List<string[]> HeaderList { get; private set; }
-
             public byte[] Content { get; private set; }
-
             public HttpStatusCode Status { get; private set;}
-
             public Dictionary<string, string> Cookies { get; private set; }
-
             public List<string> CookiesFlat { get { return Cookies.Select(c => c.Key + "=" + c.Value).ToList(); } }
-
             public string CookieHeader { get { return string.Join("; ", CookiesFlat); } }
 
             public CurlResponse(List<string[]> headers, byte[] content, HttpStatusCode s)
@@ -94,17 +96,15 @@ namespace Jackett
         {
             var curlRequest = new CurlRequest(HttpMethod.Get, url, cookies, referer);
             var result = await PerformCurlAsync(curlRequest);
-            var checkedResult = await FollowRedirect(url, result);
+            return result;
-            return checkedResult;
+        }    
-        }
 
         public static async Task<CurlResponse> PostAsync(string url, Dictionary<string, string> formData, string cookies = null, string referer = null)
         {
             var curlRequest = new CurlRequest(HttpMethod.Post, url, cookies, referer);
             curlRequest.PostData = formData;
             var result = await PerformCurlAsync(curlRequest);
-            var checkedResult = await FollowRedirect(url, result);
+            return result;
-            return checkedResult;
         }
 
         private static async Task<CurlResponse> FollowRedirect(string url, CurlResponse response)
@@ -139,78 +139,80 @@ namespace Jackett
 
         public static CurlResponse PerformCurl(CurlRequest curlRequest)
         {
-            Curl.GlobalInit(CurlInitFlag.All);
+            lock (instance)
-
-            var headerBuffers = new List<byte[]>();
-            var contentBuffers = new List<byte[]>();
-
-            using (var easy = new CurlEasy())
             {
-                easy.Url = curlRequest.Url;
+                var headerBuffers = new List<byte[]>();
-                easy.BufferSize = 64 * 1024;
+                var contentBuffers = new List<byte[]>();
-                easy.UserAgent = BrowserUtil.ChromeUserAgent;
-                easy.WriteFunction = (byte[] buf, int size, int nmemb, object data) =>
-                {
-                    contentBuffers.Add(buf);
-                    return size * nmemb;
-                };
-                easy.HeaderFunction = (byte[] buf, int size, int nmemb, object extraData) =>
-                {
-                    headerBuffers.Add(buf);
-                    return size * nmemb;
-                };
 
-                if (!string.IsNullOrEmpty(curlRequest.Cookies))
+                using (var easy = new CurlEasy())
-                    easy.Cookie = curlRequest.Cookies;
-
-                if (!string.IsNullOrEmpty(curlRequest.Referer))
-                    easy.Referer = curlRequest.Referer;
-
-                if (curlRequest.Method == HttpMethod.Post)
                 {
-                    easy.Post = true;
+                    easy.Url = curlRequest.Url;
-                    var postString = new FormUrlEncodedContent(curlRequest.PostData).ReadAsStringAsync().Result;
+                    easy.BufferSize = 64 * 1024;
-                    easy.PostFields = postString;
+                    easy.UserAgent = BrowserUtil.ChromeUserAgent;
-                    easy.PostFieldSize = Encoding.UTF8.GetByteCount(postString);
+                    easy.FollowLocation = false;
-                }
+                    easy.ConnectTimeout  = 20;
-
+                    easy.WriteFunction = (byte[] buf, int size, int nmemb, object data) =>
-                easy.Perform();
-            }
-            
-            var headerBytes = Combine(headerBuffers.ToArray());
-            var headerString = Encoding.UTF8.GetString(headerBytes);
-            var headerParts = headerString.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
-            var headers = new List<string[]>();
-            var headerCount = 0;
-            HttpStatusCode status = HttpStatusCode.InternalServerError;
-            foreach (var headerPart in headerParts)
-            {
-                if (headerCount == 0)
-                {
-                    var responseCode = int.Parse(headerPart.Split(' ')[1]);
-                    status = (HttpStatusCode)responseCode;
-                }
-                else
-                {
-                    var keyVal = headerPart.Split(new char[] { ':' }, 2);
-                    if (keyVal.Length > 1)
                     {
-                        headers.Add(new[] { keyVal[0].ToLower().Trim(), keyVal[1].Trim() });
+                        contentBuffers.Add(buf);
+                        return size * nmemb;
+                    };
+                    easy.HeaderFunction = (byte[] buf, int size, int nmemb, object extraData) =>
+                    {
+                        headerBuffers.Add(buf);
+                        return size * nmemb;
+                    };
+
+                    if (!string.IsNullOrEmpty(curlRequest.Cookies))
+                        easy.Cookie = curlRequest.Cookies;
+
+                    if (!string.IsNullOrEmpty(curlRequest.Referer))
+                        easy.Referer = curlRequest.Referer;
+
+                    if (curlRequest.Method == HttpMethod.Post)
+                    {
+                        easy.Post = true;
+                        var postString = new FormUrlEncodedContent(curlRequest.PostData).ReadAsStringAsync().Result;
+                        easy.PostFields = postString;
+                        easy.PostFieldSize = Encoding.UTF8.GetByteCount(postString);
                     }
+
+                    easy.Perform();
                 }
 
-                headerCount++;
+                var headerBytes = Combine(headerBuffers.ToArray());
+                var headerString = Encoding.UTF8.GetString(headerBytes);
+                var headerParts = headerString.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
+                var headers = new List<string[]>();
+                var headerCount = 0;
+                HttpStatusCode status = HttpStatusCode.InternalServerError;
+                foreach (var headerPart in headerParts)
+                {
+                    if (headerCount == 0)
+                    {
+                        var responseCode = int.Parse(headerPart.Split(' ')[1]);
+                        status = (HttpStatusCode)responseCode;
+                    }
+                    else
+                    {
+                        var keyVal = headerPart.Split(new char[] { ':' }, 2);
+                        if (keyVal.Length > 1)
+                        {
+                            headers.Add(new[] { keyVal[0].ToLower().Trim(), keyVal[1].Trim() });
+                        }
+                    }
+
+                    headerCount++;
+                }
+
+                var contentBytes = Combine(contentBuffers.ToArray());
+
+                var curlResponse = new CurlResponse(headers, contentBytes, status);
+                if (!string.IsNullOrEmpty(curlRequest.Cookies))
+                    curlResponse.AddCookiesFromHeaderValue(curlRequest.Cookies);
+                curlResponse.AddCookiesFromHeaders(headers);
+
+                return curlResponse;
             }
-
-            var contentBytes = Combine(contentBuffers.ToArray());
-
-            var curlResponse = new CurlResponse(headers, contentBytes, status);
-            if (!string.IsNullOrEmpty(curlRequest.Cookies))
-                curlResponse.AddCookiesFromHeaderValue(curlRequest.Cookies);
-            curlResponse.AddCookiesFromHeaders(headers);
-
-            return curlResponse;
-
         }
 
         public static byte[] Combine(params byte[][] arrays)

src/Jackett/Indexers/AlphaRatio.cs

@@ -105,7 +105,6 @@ namespace Jackett.Indexers
                 dom["#loginform > table"].Remove();
                 var errorMessage = dom["#loginform"].Text().Trim().Replace("\n\t", " ");
                 throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config);
-
             }
             else
             {

src/Jackett/Indexers/BaseIndexer.cs

@@ -24,7 +24,6 @@ namespace Jackett.Indexers
 
         protected Logger logger;
         protected IIndexerManagerService indexerService;
-       // protected IWebClient webClient;
 
         protected static List<CachedResult> cache = new List<CachedResult>();
         protected static readonly TimeSpan cacheTime = new TimeSpan(0, 9, 0);
@@ -42,7 +41,6 @@ namespace Jackett.Indexers
             TorznabCaps = caps;
             this.logger = logger;
             indexerService = manager;
-          //  webClient = wc;
         }
 
         protected void SaveConfig(JToken config)

src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs

@@ -55,6 +55,7 @@ namespace Jackett.Utils.Clients
                 }
             }
 
+            logger.Debug(string.Format("UnixLibCurlWebClient: Returning", result.Status));
             return result;
         }
 
@@ -63,12 +64,14 @@ namespace Jackett.Utils.Clients
             logger.Debug(string.Format("UnixLibCurlWebClient:GetString(Url:{0})", request.Url));
             var result = await GetBytes(request);
 
-            return new WebClientStringResult()
+            var sresult = new WebClientStringResult()
             {
                 Content = Encoding.UTF8.GetString(result.Content),
                 Cookies = result.Cookies,
                 Status = result.Status
             };
+
+            return sresult;
         }
     }
 }