Added simple (non-auth) proxy support including processing the second set of headers (server) vs first (proxy). New command line option (-j 127.0.0.1:8888) to set the proxy and port. unfortunatly both -p and -x were already taken

extended refresh header handling to libcurl and safecurl
also some minor tweaks needed to have Curl 'behave' with certain servers by adding accept-language header to Curl
Added ability/template to allow the user to select their own SiteURL (see BitSoup implementation) with a minor update to ConfigurationData class to prevent the "display text" from disappearing on errors.
XSpeeds indexer - updated to handle a case where the returned XML occasionally is null terminated resulting in XML parsing exceptions
BitSoup indexer - added user customizable url selection, including some url validation
BaseIndexer - cleaned up some of my earlier implementation for accumulating cookies.

src/Jackett.Console/ConsoleOptions.cs

@@ -27,6 +27,9 @@ namespace Jackett.Console
         [Option('c', "UseClient",  HelpText = "Override web client selection. [automatic(Default)/libcurl/safecurl/httpclient]")]
         public string Client { get; set; }
 
+        [Option('j', "ProxyConnection", HelpText = "use proxy - e.g. 127.0.0.1:8888")]
+        public string ProxyConnection { get; set; }
+
         [Option('s', "Start",  HelpText = "Start the Jacket Windows service (Must be admin)")]
         public bool StartService { get; set; }
 

src/Jackett.Console/Program.cs

@@ -65,6 +65,11 @@ namespace JackettConsole
                     if (options.Client != null)
                         Startup.ClientOverride = options.Client.ToLowerInvariant();
 
+                    // Use Proxy
+                    if (options.ProxyConnection != null)
+                    {
+                        Startup.ProxyConnection = options.ProxyConnection.ToLowerInvariant();
+                    }
                     // Logging
                     if (options.Logging)
                         Startup.LogRequests = true;

src/Jackett/CurlHelper.cs

@@ -85,6 +85,7 @@ namespace Jackett
 
                 using (var easy = new CurlEasy())
                 {
+                    
                     easy.Url = curlRequest.Url;
                     easy.BufferSize = 64 * 1024;
                     easy.UserAgent = BrowserUtil.ChromeUserAgent;
@@ -141,6 +142,11 @@ namespace Jackett
                         easy.SetOpt(CurlOption.SslVerifyPeer, false);
                     }
 
+                    if (Startup.ProxyConnection != null)
+                    {
+                        easy.SetOpt(CurlOption.Proxy, Startup.ProxyConnection);
+                    }                    
+
                     easy.Perform();
 
                     if (easy.LastErrorCode != CurlCode.Ok)
@@ -155,6 +161,15 @@ namespace Jackett
 
                 var headerBytes = Combine(headerBuffers.ToArray());
                 var headerString = Encoding.UTF8.GetString(headerBytes);
+                if (Startup.ProxyConnection != null)
+                {
+                    var firstcrlf = headerString.IndexOf("\r\n\r\n");
+                    var secondcrlf = headerString.IndexOf("\r\n\r\n", firstcrlf + 1);
+                    if (secondcrlf > 0)
+                    {
+                        headerString = headerString.Substring(firstcrlf + 4, secondcrlf - (firstcrlf));
+                    }
+                }
                 var headerParts = headerString.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
                 var headers = new List<string[]>();
                 var headerCount = 0;

src/Jackett/Indexers/BaseIndexer.cs

@@ -147,28 +147,28 @@ namespace Jackett.Indexers
             }
         }
 
-        protected async Task FollowIfRedirect(WebClientStringResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
+        protected async Task FollowIfRedirect(WebClientStringResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false)
         {
             var byteResult = new WebClientByteResult();
             // Map to byte
             Mapper.Map(response, byteResult);
-            await FollowIfRedirect(byteResult, referrer, overrideRedirectUrl, overrideCookies);
+            await FollowIfRedirect(byteResult, referrer, overrideRedirectUrl, overrideCookies, accumulateCookies);
             // Map to string
             Mapper.Map(byteResult, response);
         }
 
-        protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
+        protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false)
         {
             // Follow up  to 5 redirects
             for (int i = 0; i < 5; i++)
             {
                 if (!response.IsRedirect)
                     break;
-                await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies);
+                await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies, accumulateCookies);
-                if (response.Cookies != null && overrideCookies != null)
+                if (accumulateCookies)
                 {
-                    response.Cookies = overrideCookies + " " + response.Cookies;
+                    CookieHeader = ResolveCookies(response.Cookies);
-                    overrideCookies = response.Cookies;
+                    response.Cookies = CookieHeader;
                 }
                 if (overrideCookies != null && response.Cookies == null)
                 {
@@ -177,16 +177,39 @@ namespace Jackett.Indexers
             }
         }
 
-        private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
+        private String ResolveCookies(String incomingCookies = "")
+        {
+            var redirRequestCookies = (CookieHeader != "" ? CookieHeader + " " : "") + incomingCookies;
+            System.Text.RegularExpressions.Regex expression = new System.Text.RegularExpressions.Regex(@"([^\s]+)=([^=]+)(?:\s|$)");
+            Dictionary<string, string> cookieDIctionary = new Dictionary<string, string>();
+            var matches = expression.Match(redirRequestCookies);
+            while (matches.Success)
+            {
+                if (matches.Groups.Count > 2) cookieDIctionary[matches.Groups[1].Value] = matches.Groups[2].Value;
+                matches = matches.NextMatch();
+            }
+            return string.Join(" ", cookieDIctionary.Select(kv => kv.Key.ToString() + "=" + kv.Value.ToString()).ToArray());
+            
+        }
+
+        private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false)
         {
             if (incomingResponse.IsRedirect)
             {
+                var redirRequestCookies = "";
+                if (accumulateCookies)
+                {
+                    redirRequestCookies = ResolveCookies((CookieHeader != "" ? CookieHeader + " " : "") + (overrideCookies != null ? overrideCookies : ""));
+                } else
+                {
+                    redirRequestCookies = (overrideCookies != null ? overrideCookies : "");
+                }
                 // Do redirect
                 var redirectedResponse = await webclient.GetBytes(new WebRequest()
                 {
                     Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo,
                     Referer = referrer,
-                    Cookies = overrideCookies ?? CookieHeader
+                    Cookies = redirRequestCookies
                 });
                 Mapper.Map(redirectedResponse, incomingResponse);
             }
@@ -371,29 +394,19 @@ namespace Jackett.Indexers
             var response = await webclient.GetString(request);
             if (accumulateCookies)
             {
-                response.Cookies = (request.Cookies == null ? "" : request.Cookies + " ") + response.Cookies;
+                response.Cookies = ResolveCookies((request.Cookies == null ? "" : request.Cookies + " ") + response.Cookies);
             }
             var firstCallCookies = response.Cookies;
 
             if (response.IsRedirect)
             {
-                await FollowIfRedirect(response, request.Url, redirectUrlOverride, response.Cookies);
+                await FollowIfRedirect(response, request.Url, redirectUrlOverride, response.Cookies, accumulateCookies);
             }
 
             if (returnCookiesFromFirstCall)
             {
-                response.Cookies = firstCallCookies + (accumulateCookies ? " " + response.Cookies : "");
+                response.Cookies = ResolveCookies(firstCallCookies + (accumulateCookies ? " " + response.Cookies : ""));
             }
-            // resolve cookie conflicts - really no need for this as the webclient will handle it
-            System.Text.RegularExpressions.Regex expression = new System.Text.RegularExpressions.Regex(@"([^\s]+)=([^=]+)(?:\s|$)");
-            Dictionary<string, string> cookieDIctionary = new Dictionary<string, string>();
-            var matches = expression.Match(response.Cookies);
-            while (matches.Success)
-            {
-                if (matches.Groups.Count > 2) cookieDIctionary[matches.Groups[1].Value] = matches.Groups[2].Value;
-                matches = matches.NextMatch();
-            }
-            response.Cookies = string.Join(" ", cookieDIctionary.Select(kv => kv.Key.ToString() + "=" + kv.Value.ToString()).ToArray());
             
             return response;
         }

src/Jackett/Indexers/BitSoup.cs

@@ -7,7 +7,6 @@ using Newtonsoft.Json.Linq;
 using NLog;
 using System;
 using System.Collections.Generic;
-using System.Collections.Specialized;
 using System.Globalization;
 using System.Linq;
 using System.Net;
@@ -16,17 +15,22 @@ using System.Text;
 using System.Threading.Tasks;
 using System.Web;
 using Jackett.Models.IndexerConfig;
+using System.Collections.Specialized;
+using System.Text.RegularExpressions;
 
 namespace Jackett.Indexers
 {
     public class BitSoup : BaseIndexer, IIndexer
     {
-        private string BrowseUrl { get { return SiteLink + "browse.php"; } }
+        private string UseLink { get { return (this.configData.AlternateLink.Value != "" ? this.configData.AlternateLink.Value : SiteLink); } }
-        private string LoginUrl { get { return SiteLink + "takelogin.php"; } }
+        private string BrowseUrl { get { return UseLink + "browse.php"; } }
+        private string LoginUrl { get { return UseLink + "takelogin.php"; } }
+        private string LoginReferer {  get { return UseLink + "login.php";  } }
+        private List<String> KnownURLs = new List<String>{ "https://www.bitsoup.me/","https://www.bitsoup.org/"};
 
-        new ConfigurationDataBasicLogin configData
+        new NxtGnConfigurationData configData
         {
-            get { return (ConfigurationDataBasicLogin)base.configData; }
+            get { return (NxtGnConfigurationData)base.configData; }
             set { base.configData = value; }
         }
 
@@ -39,8 +43,11 @@ namespace Jackett.Indexers
                 client: wc,
                 logger: l,
                 p: ps,
-                configData: new ConfigurationDataBasicLogin())
+                configData: new NxtGnConfigurationData())
         {
+            this.configData.DisplayText.Value = this.DisplayName + " has multiple URLs.  The default (" + this.SiteLink + ") can be changed by entering a new value in the box below.";
+            this.configData.DisplayText.Value += "The following are some known URLs for " + this.DisplayName;
+            this.configData.DisplayText.Value += "<ul><li>" + String.Join("</li><li>", this.KnownURLs.ToArray()) + "</li></ul>";
 
             //AddCategoryMapping("624", TorznabCatType.Console);
             //AddCategoryMapping("307", TorznabCatType.ConsoleNDS);
@@ -139,20 +146,33 @@ namespace Jackett.Indexers
         public async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
         {
             configData.LoadValuesFromJson(configJson);
+            if (configData.AlternateLink.Value != null && configData.AlternateLink.Value != "")
+            {
+                if (!configData.AlternateLink.Value.EndsWith("/"))
+                {
+                    configData.AlternateLink.Value = null;
+                    throw new Exception("AlternateLink must end with a slash.");
+                }
+                var match = Regex.Match(configData.AlternateLink.Value, "^https?:\\/\\/(?:[\\w]+\\.)+(?:[a-zA-Z]+)\\/$");
+                if (!match.Success)
+                {
+                    configData.AlternateLink.Value = null;
+                    throw new Exception("AlternateLink must be a valid url.");
+                }              
+            }
             var pairs = new Dictionary<string, string> {
                 { "username", configData.Username.Value },
                 { "password", configData.Password.Value },
-                { "returnto", "/" },
+                
-                { "login", "Log in!" }
             };
 
-            var loginPage = await RequestStringWithCookies(SiteLink, string.Empty);
+            var loginPage = await RequestStringWithCookies(UseLink, string.Empty);
 
-            var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, SiteLink, SiteLink, true);
+            var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, null, LoginReferer, true);
             await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("logout.php"), () =>
             {
                 CQ dom = result.Content;
-                var messageEl = dom["body > div"].First();
+                var messageEl = dom["body > table.statusbar1 > tbody > tr > td > table > tbody > tr > td > table > tbody > tr > td"].First();
                 var errorMessage = messageEl.Text().Trim();
                 throw new ExceptionWithConfigData(errorMessage, configData);
             });
@@ -218,9 +238,9 @@ namespace Jackett.Indexers
                     var release = new ReleaseInfo();
 
                     release.Title = row.Cq().Find("td:eq(1) a").First().Text().Trim();
-                    release.Comments = new Uri(SiteLink + row.Cq().Find("td:eq(1) a").First().Attr("href"));
+                    release.Comments = new Uri(UseLink + row.Cq().Find("td:eq(1) a").First().Attr("href"));
 
-                    release.Link = new Uri(SiteLink + row.Cq().Find("td:eq(2) a").First().Attr("href"));
+                    release.Link = new Uri(UseLink + row.Cq().Find("td:eq(2) a").First().Attr("href"));
                     release.Description = release.Title;
                     var cat = row.Cq().Find("td:eq(0) a").First().Attr("href").Substring(15);
                     release.Category = MapTrackerCatToNewznab(cat);
@@ -242,5 +262,23 @@ namespace Jackett.Indexers
                 OnParseError(results, ex);
             }
         }
+
+        public class NxtGnConfigurationData : ConfigurationData
+        {
+            public StringItem Username { get; private set; }
+            public StringItem Password { get; private set; }
+            public DisplayItem DisplayText { get; private set; }
+            public StringItem AlternateLink { get; set; }
+           
+
+            public NxtGnConfigurationData()
+            {
+                Username = new StringItem { Name = "Username" };
+                Password = new StringItem { Name = "Password" };
+                DisplayText = new DisplayItem("") { Name = "" };
+                AlternateLink = new StringItem { Name = "AlternateLinks" };
+            }
+            
+        }
     }
 }

src/Jackett/Indexers/XSpeeds.cs

@@ -127,6 +127,9 @@ namespace Jackett.Indexers
             if (string.IsNullOrWhiteSpace(searchString))
             {
                 var rssPage = await RequestStringWithCookiesAndRetry(string.Format(RSSUrl, configData.RSSKey.Value));
+                if (rssPage.Content.EndsWith("\0")) {
+                    rssPage.Content = rssPage.Content.Substring(0, rssPage.Content.Length - 1);
+                }
                 var rssDoc = XDocument.Parse(rssPage.Content);
 
                 foreach (var item in rssDoc.Descendants("item"))
@@ -170,6 +173,11 @@ namespace Jackett.Indexers
             }
             else
             {
+                if (searchString.Length < 3)
+                {
+                    OnParseError("", new Exception("Minimum search length is 3"));
+                    return releases;
+                }
                 var searchParams = new Dictionary<string, string> {
                     { "do", "search" },
                     { "keywords",  searchString },

src/Jackett/Models/IndexerConfig/ConfigurationData.cs

@@ -132,7 +132,7 @@ namespace Jackett.Models.IndexerConfig
             if (!forDisplay)
             {
                 properties = properties
-                    .Where(p => p.ItemType == ItemType.HiddenData || p.ItemType == ItemType.InputBool || p.ItemType == ItemType.InputString || p.ItemType == ItemType.Recaptcha)
+                    .Where(p => p.ItemType == ItemType.HiddenData || p.ItemType == ItemType.InputBool || p.ItemType == ItemType.InputString || p.ItemType == ItemType.Recaptcha || p.ItemType == ItemType.DisplayInfo)
                     .ToArray();
             }
 

src/Jackett/Startup.cs

@@ -40,6 +40,11 @@ namespace Jackett
             set;
         }
 
+        public static string ProxyConnection
+        {
+            get;
+            set;
+        }
         public static bool? DoSSLFix
         {
             get;

src/Jackett/Utils/Clients/HttpWebClient.cs

@@ -62,18 +62,28 @@ namespace Jackett.Utils.Clients
                     }
                 }
             }
-
+            var useProxy = false;
+            WebProxy proxyServer = null;
+            if (Startup.ProxyConnection != null)
+            {
+                proxyServer = new WebProxy(Startup.ProxyConnection, false);
+                useProxy = true;
+            }
             var client = new HttpClient(new HttpClientHandler
             {
                 CookieContainer = cookies,
                 AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
                 UseCookies = true,
+                Proxy = proxyServer,
+                UseProxy = useProxy
             });
             
-            if(webRequest.EmulateBrowser)
+
+            if (webRequest.EmulateBrowser)
                 client.DefaultRequestHeaders.Add("User-Agent",  BrowserUtil.ChromeUserAgent);
             else
                 client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
+            
             HttpResponseMessage response = null;
             var request = new HttpRequestMessage();
             request.Headers.ExpectContinue = false;
@@ -158,6 +168,7 @@ namespace Jackett.Utils.Clients
             // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
             IEnumerable<string> cookieHeaders;
             var responseCookies = new List<Tuple<string, string>>();
+
             if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
             {
                 foreach (var value in cookieHeaders)

src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs

@@ -104,6 +104,28 @@ namespace Jackett.Utils.Clients
                         case "location":
                             result.RedirectingTo = header[1];
                             break;
+                        case "refresh":
+                            if (response.Status == System.Net.HttpStatusCode.ServiceUnavailable)
+                            {
+                                //"Refresh: 8;URL=/cdn-cgi/l/chk_jschl?pass=1451000679.092-1vJFUJLb9R"
+                                var redirval = "";
+                                var value = header[1];
+                                var start = value.IndexOf("=");
+                                var end = value.IndexOf(";");
+                                var len = value.Length;
+                                if (start > -1)
+                                {
+                                    redirval = value.Substring(start + 1);
+                                    result.RedirectingTo = redirval;
+                                    // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
+                                    // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
+                                    // it shoudln't include service unavailable..only if we have this redirect header.
+                                    result.Status = System.Net.HttpStatusCode.Redirect;
+                                    var redirtime = Int32.Parse(value.Substring(0, end));
+                                    System.Threading.Thread.Sleep(redirtime * 1000);
+                                }
+                            }
+                            break;
                     }
                 }
             }

src/Jackett/Utils/Clients/UnixSafeCurlWebClient.cs

@@ -50,6 +50,11 @@ namespace Jackett.Utils.Clients
         private async Task<WebClientByteResult> Run(WebRequest request)
         {
             var args = new StringBuilder();
+            if (Startup.ProxyConnection != null)
+            {
+                args.AppendFormat("-x " + Startup.ProxyConnection + " ");
+            }
+            
             args.AppendFormat("--url \"{0}\" ", request.Url);
            
             if (request.EmulateBrowser)
@@ -86,11 +91,16 @@ namespace Jackett.Utils.Clients
                 // https://git.fedorahosted.org/cgit/mod_nss.git/plain/docs/mod_nss.html
                 args.Append("--cipher " + SSLFix.CipherList);
             }
-
+            if (Startup.IgnoreSslErrors == true)
+            {
+                args.Append("-k ");
+            }
+            args.Append("-H \"Accept-Language: en-US,en\" ");
+            args.Append("-H \"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\" ");
             string stdout = null;
             await Task.Run(() =>
             {
-                stdout = processService.StartProcessAndGetOutput(System.Environment.OSVersion.Platform == PlatformID.Unix ? "curl" : "curl.exe", args.ToString(), true);
+                stdout = processService.StartProcessAndGetOutput(System.Environment.OSVersion.Platform == PlatformID.Unix ? "curl" : "curl.exe", args.ToString() , true);
             });
 
             var outputData = File.ReadAllBytes(tempFile);
@@ -101,6 +111,16 @@ namespace Jackett.Utils.Clients
             if (headSplit < 0)
                 throw new Exception("Invalid response");
             var headers = stdout.Substring(0, headSplit);
+            if (Startup.ProxyConnection != null)
+            {
+                // the proxy provided headers too so we need to split headers again
+                var headSplit1 = stdout.IndexOf("\r\n\r\n",headSplit + 4);
+                if (headSplit1 > 0)
+                {
+                    headers = stdout.Substring(headSplit + 4,headSplit1 - (headSplit + 4));
+                    headSplit = headSplit1;
+                }
+            }
             var headerCount = 0;
             var cookieBuilder = new StringBuilder();
             var cookies = new List<Tuple<string, string>>();
@@ -131,6 +151,24 @@ namespace Jackett.Utils.Clients
                             case "location":
                                 result.RedirectingTo = value.Trim();
                                 break;
+                            case "refresh":
+                                //"Refresh: 8;URL=/cdn-cgi/l/chk_jschl?pass=1451000679.092-1vJFUJLb9R"
+                                var redirval = "";
+                                var start = value.IndexOf("=");
+                                var end = value.IndexOf(";");
+                                var len = value.Length;
+                                if (start > -1)
+                                {
+                                    redirval = value.Substring(start + 1);
+                                    result.RedirectingTo = redirval;
+                                    // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
+                                    // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
+                                    // it shoudln't include service unavailable..only if we have this redirect header.
+                                    result.Status = System.Net.HttpStatusCode.Redirect;
+                                    var redirtime = Int32.Parse(value.Substring(0, end));
+                                    System.Threading.Thread.Sleep(redirtime * 1000);
+                                }
+                                break;
                         }
                     }
                 }