Complete Shazbat, fix category picker on web ui and start btn

README.md

@@ -40,7 +40,7 @@ We were previously focused on TV but are working on extending searches to allow
  * [NextGen](https://nxtgn.org/)
  * [pretome](https://pretome.info)
  * [PrivateHD](https://privatehd.to/)
- * [RARGB](https://rarbg.to/)
+ * [RARBG](https://rarbg.to/)
  * [RuTor](http://rutor.org/)
  * [SceneAccess](https://sceneaccess.eu/login)
  * [SceneTime](https://www.scenetime.com/)

src/Jackett/Content/custom.js

@@ -367,7 +367,7 @@ function bindUIButtons() {
                          var count = 0;
                          this.api().columns().every(function () {
                              count++;
-                             if (count === 5 || count === 7) {
+                             if (count === 5 || count === 9) {
                                  var column = this;
                                  var select = $('<select><option value=""></option></select>')
                                      .appendTo($(column.footer()).empty())
@@ -495,7 +495,7 @@ function bindUIButtons() {
                             var count = 0;
                             this.api().columns().every(function () {
                                 count++;
-                                if (count === 3 || count === 5) {
+                                if (count === 3 || count === 7) {
                                     var column = this;
                                     var select = $('<select><option value=""></option></select>')
                                         .appendTo($(column.footer()).empty())

src/Jackett/Indexers/AnimeBytes.cs

@@ -1,6 +1,7 @@
 using CsQuery;
 using Jackett.Models;
 using Jackett.Models.IndexerConfig;
+using Jackett.Models.IndexerConfig.Bespoke;
 using Jackett.Services;
 using Jackett.Utils;
 using Jackett.Utils.Clients;

src/Jackett/Indexers/BaseIndexer.cs

@@ -311,7 +311,7 @@ namespace Jackett.Indexers
             return await webclient.GetBytes(request);
         }
 
-        protected async Task<WebClientStringResult> PostDataWithCookies(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null)
+        protected async Task<WebClientStringResult> PostDataWithCookies(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null)
         {
             var request = new Utils.Clients.WebRequest()
             {
@@ -319,19 +319,20 @@ namespace Jackett.Indexers
                 Type = RequestType.POST,
                 Cookies = cookieOverride ?? CookieHeader,
                 PostData = data,
-                Referer = referer
+                Referer = referer,
+                Headers = headers
             };
             return await webclient.GetString(request);
         }
 
-        protected async Task<WebClientStringResult> PostDataWithCookiesAndRetry(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null)
+        protected async Task<WebClientStringResult> PostDataWithCookiesAndRetry(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null)
         {
             Exception lastException = null;
             for (int i = 0; i < 3; i++)
             {
                 try
                 {
-                    return await PostDataWithCookies(url, data, cookieOverride, referer);
+                    return await PostDataWithCookies(url, data, cookieOverride, referer, headers, rawbody);
                 }
                 catch (Exception e)
                 {

src/Jackett/Indexers/BroadcastTheNet.cs

@@ -0,0 +1,151 @@
+using CsQuery;
+using Jackett.Models;
+using Jackett.Services;
+using Jackett.Utils;
+using Jackett.Utils.Clients;
+using Newtonsoft.Json.Linq;
+using NLog;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Text;
+using System.Threading.Tasks;
+using System.Web;
+using Jackett.Models.IndexerConfig;
+using System.Dynamic;
+
+namespace Jackett.Indexers
+{
+    public class BroadcastTheNet : BaseIndexer, IIndexer
+    {
+        string APIBASE = "http://api.btnapps.net/";
+
+        new ConfigurationDataAPIKey configData
+        {
+            get { return (ConfigurationDataAPIKey)base.configData; }
+            set { base.configData = value; }
+        }
+
+        public BroadcastTheNet(IIndexerManagerService i, IWebClient wc, Logger l, IProtectionService ps)
+            : base(name: "BroadcastTheNet",
+                description: "Needs no description..",
+                link: "https://broadcasthe.net/",
+                caps: TorznabUtil.CreateDefaultTorznabTVCaps(),
+                manager: i,
+                client: wc,
+                logger: l,
+                p: ps,
+                configData: new ConfigurationDataAPIKey())
+        {
+        }
+
+        public async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
+        {
+            configData.LoadValuesFromJson(configJson);
+
+            IsConfigured = false;
+            try
+            {
+                var results = await PerformQuery(new TorznabQuery());
+                if (results.Count() == 0)
+                    throw new Exception("Testing returned no results!");
+                IsConfigured = true;
+                SaveConfig();
+            }
+            catch(Exception e)
+            {
+                throw new ExceptionWithConfigData(e.Message, configData);
+            }
+
+            return IndexerConfigurationStatus.Completed;
+        }
+
+
+        private string JsonRPCRequest(string method, dynamic parameters)
+        {
+            dynamic request = new ExpandoObject();
+            request["jsonrpc"] = "2.0";
+            request["method"] = method;
+            request["params"] = parameters;
+            request["id"] = Guid.NewGuid().ToString().Substring(0, 8);
+            return request.ToString();
+        }
+
+        public async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
+        {
+            var releases = new List<ReleaseInfo>();
+
+
+            var response = await PostDataWithCookiesAndRetry(APIBASE, null, null, null, new Dictionary<string, string>()
+            {
+                { "Accept", "application/json-rpc, application/json"},
+                {"ContentType", "application/json-rpc"}
+            }, JsonRPCRequest("getTorrents", new Object[]
+            {
+                configData.Key.Value
+            }));
+
+            /* 
+             var searchUrl = BrowsePage;
+
+             if (!string.IsNullOrWhiteSpace(query.GetQueryString()))
+             {
+                 searchUrl += string.Format(QueryString, HttpUtility.UrlEncode(query.GetQueryString()));
+             }
+
+             var results = await RequestStringWithCookiesAndRetry(searchUrl);
+
+             try
+             {
+                 CQ dom = results.Content;
+
+                 var rows = dom["#sortabletable tr"];
+                 foreach (var row in rows.Skip(1))
+                 {
+                     var release = new ReleaseInfo();
+                     var qRow = row.Cq();
+                     release.Title = qRow.Find(".tooltip-content div").First().Text();
+                     if (string.IsNullOrWhiteSpace(release.Title))
+                         continue;
+                     release.Description = qRow.Find(".tooltip-content div").Get(1).InnerText.Trim();
+
+                     var qLink = row.Cq().Find("td:eq(2) a:eq(1)");
+                     release.Link = new Uri(qLink.Attr("href"));
+                     release.Guid = release.Link;
+                     release.Comments = new Uri(qRow.Find(".tooltip-target a").First().Attr("href"));
+
+                     // 07-22-2015 11:08 AM
+                     var dateString = qRow.Find("td:eq(1) div").Last().Children().Remove().End().Text().Trim();
+                     release.PublishDate = DateTime.ParseExact(dateString, "MM-dd-yyyy hh:mm tt", CultureInfo.InvariantCulture);
+
+                     var sizeStr = qRow.Find("td:eq(4)").Text().Trim();
+                     release.Size = ReleaseInfo.GetBytes(sizeStr);
+
+                     release.Seeders = ParseUtil.CoerceInt(qRow.Find("td:eq(6)").Text().Trim());
+                     release.Peers = ParseUtil.CoerceInt(qRow.Find("td:eq(7)").Text().Trim()) + release.Seeders;
+
+                     var catLink = row.Cq().Find("td:eq(0) a").First().Attr("href");
+                     var catSplit = catLink.IndexOf("category=");
+                     if (catSplit > -1)
+                     {
+                         catLink = catLink.Substring(catSplit + 9);
+                     }
+
+                     release.Category = MapTrackerCatToNewznab(catLink);
+                     releases.Add(release);
+                 }
+             }
+             catch (Exception ex)
+             {
+                 OnParseError(results.Content, ex);
+             }*/
+
+            return releases;
+        }
+
+
+    }
+}

src/Jackett/Indexers/FileList.cs

@@ -16,6 +16,7 @@ using System.Text;
 using System.Threading.Tasks;
 using System.Web;
 using Jackett.Models.IndexerConfig;
+using Jackett.Models.IndexerConfig.Bespoke;
 
 namespace Jackett.Indexers
 {

src/Jackett/Indexers/RUTor.cs

@@ -17,6 +17,7 @@ using System.Text.RegularExpressions;
 using Jackett.Models.IndexerConfig;
 using System.Globalization;
 using Newtonsoft.Json;
+using Jackett.Models.IndexerConfig.Bespoke;
 
 namespace Jackett.Indexers
 {

src/Jackett/Indexers/Shazbat.cs

@@ -17,6 +17,8 @@ using System.Web;
 using Jackett.Models.IndexerConfig;
 using System.Globalization;
 using System.Text.RegularExpressions;
+using System.Xml.Linq;
+using System.Xml.XPath;
 
 namespace Jackett.Indexers
 {
@@ -25,10 +27,11 @@ namespace Jackett.Indexers
         private string LoginUrl { get { return SiteLink + "login"; } }
         private string SearchUrl { get { return SiteLink + "search"; } }
         private string TorrentsUrl { get { return SiteLink + "torrents"; } }
+        private string RSSProfile { get { return SiteLink + "rss_feeds"; } }
 
-        new ConfigurationDataBasicLogin configData
+        new ConfigurationDataBasicLoginWithRSS configData
         {
-            get { return (ConfigurationDataBasicLogin)base.configData; }
+            get { return (ConfigurationDataBasicLoginWithRSS)base.configData; }
             set { base.configData = value; }
         }
 
@@ -43,7 +46,7 @@ namespace Jackett.Indexers
                 client: c,
                 logger: l,
                 p: ps,
-                configData: new ConfigurationDataBasicLogin())
+                configData: new ConfigurationDataBasicLoginWithRSS())
         {
         }
 
@@ -67,6 +70,15 @@ namespace Jackett.Indexers
                 throw new ExceptionWithConfigData("The username and password entered do not match.", configData);
             });
 
+            var rssProfile = await RequestStringWithCookiesAndRetry(RSSProfile);
+            CQ rssDom = rssProfile.Content;
+            configData.RSSKey.Value = rssDom.Find(".col-sm-9:eq(0)").Text().Trim();
+            if (string.IsNullOrWhiteSpace(configData.RSSKey.Value))
+            {
+                throw new ExceptionWithConfigData("Failed to find RSS key.", configData);
+            }
+
+            SaveConfig();
             return IndexerConfigurationStatus.RequiresTesting;
         }
 
@@ -87,64 +99,115 @@ namespace Jackett.Indexers
                 };
 
                 results = await PostDataWithCookiesAndRetry(SearchUrl, pairs, null, TorrentsUrl);
+
+                try
+                {
+                    CQ dom = results.Content;
+                    var rows = dom["#torrent-table tr"];
+
+                    if (!string.IsNullOrWhiteSpace(queryString))
+                    {
+                        rows = dom["table tr"];
+                    }
+
+                    foreach (var row in rows.Skip(1))
+                    {
+                        var release = new ReleaseInfo();
+                        var qRow = row.Cq();
+                        var titleRow = qRow.Find("td:eq(2)").First();
+                        titleRow.Children().Remove();
+                        release.Title = titleRow.Text().Trim();
+                        if (string.IsNullOrWhiteSpace(release.Title))
+                            continue;
+                        release.Description = release.Title;
+
+                        var qLink = row.Cq().Find("td:eq(4) a:eq(0)");
+                        release.Link = new Uri(SiteLink + qLink.Attr("href"));
+                        release.Guid = release.Link;
+                        var qLinkComm = row.Cq().Find("td:eq(4) a:eq(1)");
+                        release.Comments = new Uri(SiteLink + qLinkComm.Attr("href"));
+
+                        var dateString = qRow.Find(".datetime").Attr("data-timestamp");
+                        release.PublishDate = DateTimeUtil.UnixTimestampToDateTime(ParseUtil.CoerceDouble(dateString));
+                        var infoString = row.Cq().Find("td:eq(3)").Text();
+
+                        release.Size = ParseUtil.CoerceLong(Regex.Match(infoString, "\\((\\d+)\\)").Value.Replace("(", "").Replace(")", ""));
+
+                        var infosplit = infoString.Replace("/", string.Empty).Split(":".ToCharArray());
+                        release.Seeders = ParseUtil.CoerceInt(infosplit[1]);
+                        release.Peers = release.Seeders + ParseUtil.CoerceInt(infosplit[2]);
+
+                        // var tags = row.Cq().Find(".label-tag").Text(); These don't see to parse - bad tags?
+
+                        releases.Add(release);
+                    }
+                }
+                catch (Exception ex)
+                {
+                    OnParseError(results.Content, ex);
+                }
             }
             else
             {
-                var pairs = new Dictionary<string, string> {
+                var rssUrl = SiteLink + "rss/recent?passkey=" + configData.RSSKey.Value;
-                    { "portlet", "true"}
-                };
 
-                results = await PostDataWithCookiesAndRetry(TorrentsUrl, pairs, null, TorrentsUrl);
+                results = await RequestStringWithCookiesAndRetry(rssUrl);
-            }
+                try
-
-            try
-            {
-                CQ dom = results.Content;
-                var rows = dom["#torrent-table tr"];
-                foreach (var row in rows.Skip(1))
                 {
-                    var release = new ReleaseInfo();
+                    var doc = XDocument.Parse(results.Content);
-                    var qRow = row.Cq();
+                    foreach (var result in doc.Descendants("item"))
-                    var titleRow = qRow.Find("td:eq(2)").First();
-                    titleRow.Children().Remove();
-                    release.Title = titleRow.Text().Trim();
-                    if (string.IsNullOrWhiteSpace(release.Title))
-                        continue;
-                    release.Description = release.Title;
-
-                    var qLink = row.Cq().Find("td:eq(4) a:eq(0)");
-                    release.Link = new Uri(SiteLink + qLink.Attr("href"));
-                    release.Guid = release.Link;
-                    var qLinkComm = row.Cq().Find("td:eq(4) a:eq(1)");
-                    release.Comments = new Uri(SiteLink + qLinkComm.Attr("href"));
-
-                    // 07-22-2015 11:08 AM
-                    var dateString = qRow.Find(".datetime").Attr("data-timestamp");
-                    release.PublishDate = DateTimeUtil.UnixTimestampToDateTime(ParseUtil.CoerceDouble(dateString));
-                    var infoString = row.Cq().Find("td:eq(3)").Text();
-
-                    release.Size = ParseUtil.CoerceLong(Regex.Match(infoString, "\\((\\d+)\\)").Value.Replace("(","").Replace(")", ""));
-
-                    var infosplit = infoString.Replace("/", string.Empty).Split(":".ToCharArray());
-                    release.Seeders = ParseUtil.CoerceInt(infosplit[1]);
-                    release.Peers = release.Seeders +  ParseUtil.CoerceInt(infosplit[2]);
-
-                   // var tags = row.Cq().Find(".label-tag").Text(); These don't see to parse - bad tags?
-
-                    if(release.Title.Contains("1080p") || release.Title.Contains("720p"))
                     {
-                        release.Category = TorznabCatType.TVHD.ID;
+                        var xTitle = result.Element("title").Value;
-                    } else
+                        var xLink = result.Element("link").Value;
-                    {
+                        var xGUID = result.Element("guid").Value;
-                        release.Category = TorznabCatType.TVSD.ID;
+                        var xDesc = result.Element("description").Value;
+                        var xDate = result.Element("pubDate").Value;
+                        var release = new ReleaseInfo();
+                        release.Guid  =release.Link =  new Uri(xLink);
+                        release.MinimumRatio = 1;
+                        release.Seeders = 1; // We are not supplied with peer info so just mark it as one.
+                        foreach (var element in xDesc.Split(";".ToCharArray()))
+                        {
+                            var split = element.IndexOf(':');
+                            if (split > -1)
+                            {
+                                var key = element.Substring(0, split).Trim();
+                                var value = element.Substring(split+1).Trim();
+
+                                switch (key)
+                                {
+                                    case "Filename":
+                                        release.Title = release.Description = value;
+                                        break;
+                                }
+                            }
+                        }
+
+                        //"Thu, 24 Sep 2015 18:07:07 +0000"
+                        release.PublishDate = DateTime.ParseExact(xDate, "ddd, dd MMM yyyy HH:mm:ss +0000", CultureInfo.InvariantCulture);
+
+                        if (!string.IsNullOrWhiteSpace(release.Title))
+                        {
+                            releases.Add(release);
+                        }
                     }
-
+                }
-                    releases.Add(release);
+                catch (Exception ex)
+                {
+                    OnParseError(results.Content, ex);
                 }
             }
-            catch (Exception ex)
+
+            foreach(var release in releases)
             {
-                OnParseError(results.Content, ex);
+                if (release.Title.Contains("1080p") || release.Title.Contains("720p"))
+                {
+                    release.Category = TorznabCatType.TVHD.ID;
+                }
+                else
+                {
+                    release.Category = TorznabCatType.TVSD.ID;
+                }
             }
 
             return releases;

src/Jackett/Indexers/Strike.cs

@@ -15,6 +15,7 @@ using System.Threading.Tasks;
 using System.Web;
 using Jackett.Models.IndexerConfig;
 using System.Collections.Specialized;
+using Jackett.Models.IndexerConfig.Bespoke;
 
 namespace Jackett.Indexers
 {

src/Jackett/Indexers/nCore.cs

@@ -16,6 +16,7 @@ using System.Text;
 using System.Threading.Tasks;
 using System.Web;
 using Jackett.Models.IndexerConfig;
+using Jackett.Models.IndexerConfig.Bespoke;
 
 namespace Jackett.Indexers
 {

src/Jackett/Jackett.csproj

@@ -185,6 +185,7 @@
     <Compile Include="Indexers\BitMeTV.cs" />
     <Compile Include="Indexers\Demonoid.cs" />
     <Compile Include="Indexers\FrenchTorrentDb.cs" />
+    <Compile Include="Indexers\BroadcastTheNet.cs" />
     <Compile Include="Indexers\Shazbat.cs" />
     <Compile Include="Indexers\NxtGn.cs" />
     <Compile Include="Indexers\Freshon.cs" />
@@ -195,6 +196,7 @@
     <Compile Include="Indexers\FileList.cs" />
     <Compile Include="Indexers\Abstract\AvistazTracker.cs" />
     <Compile Include="Indexers\AnimeTorrents.cs" />
+    <Compile Include="Models\IndexerConfig\ConfigurationDataAPIKey.cs" />
     <Compile Include="Models\ManualSearchResult.cs" />
     <Compile Include="Indexers\Rarbg.cs" />
     <Compile Include="Indexers\TVChaosUK.cs" />
@@ -222,14 +224,14 @@
     <Compile Include="Models\CategoryMapping.cs" />
     <Compile Include="Models\AdminSearch.cs" />
     <Compile Include="Models\IndexerConfigurationStatus.cs" />
-    <Compile Include="Models\IndexerConfig\ConfigurationDataFileList.cs" />
+    <Compile Include="Models\IndexerConfig\Bespoke\ConfigurationDataFileList.cs" />
     <Compile Include="Models\IndexerConfig\ConfigurationDataBasicLoginWithRSS.cs" />
     <Compile Include="Models\IndexerConfig\ConfigurationDataRecaptchaLogin.cs" />
     <Compile Include="Models\IndexerConfig\ConfigurationDataLoginTokin.cs" />
-    <Compile Include="Models\IndexerConfig\ConfigurationDataNCore.cs" />
+    <Compile Include="Models\IndexerConfig\Bespoke\ConfigurationDataNCore.cs" />
     <Compile Include="Models\IndexerConfig\ConfigurationDataCaptchaLogin.cs" />
-    <Compile Include="Models\IndexerConfig\ConfigurationDataAnimeBytes.cs" />
+    <Compile Include="Models\IndexerConfig\Bespoke\ConfigurationDataAnimeBytes.cs" />
-    <Compile Include="Models\IndexerConfig\ConfigurationDataStrike.cs" />
+    <Compile Include="Models\IndexerConfig\Bespoke\ConfigurationDataStrike.cs" />
     <Compile Include="Models\IndexerConfig\ConfigurationDataUrl.cs" />
     <Compile Include="Models\IndexerConfig\ISerializableConfig.cs" />
     <Compile Include="Models\IndexerConfig\ConfigurationDataPinNumber.cs" />
@@ -265,7 +267,7 @@
     <Compile Include="Models\IndexerConfig\ConfigurationData.cs" />
     <Compile Include="Models\IndexerConfig\ConfigurationDataBasicLogin.cs" />
     <Compile Include="Models\IndexerConfig\ConfigurationDataCookie.cs" />
-    <Compile Include="Models\IndexerConfig\ConfigurationDataRuTor.cs" />
+    <Compile Include="Models\IndexerConfig\Bespoke\ConfigurationDataRuTor.cs" />
     <Compile Include="Controllers\AdminController.cs" />
     <Compile Include="CookieContainerExtensions.cs" />
     <Compile Include="Utils\Clients\WebRequest.cs" />
@@ -429,6 +431,9 @@
     <Content Include="Content\logos\beyondhd.png">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="Content\logos\broadcastthenet.png">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="Content\logos\demonoid.png">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Jackett/Models/IndexerConfig/Bespoke/ConfigurationDataAnimeBytes.cs

@@ -5,7 +5,7 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 
-namespace Jackett.Models.IndexerConfig
+namespace Jackett.Models.IndexerConfig.Bespoke
 {
     class ConfigurationDataAnimeBytes : ConfigurationDataBasicLogin
     {

src/Jackett/Models/IndexerConfig/Bespoke/ConfigurationDataFileList.cs

@@ -5,7 +5,7 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 
-namespace Jackett.Models.IndexerConfig
+namespace Jackett.Models.IndexerConfig.Bespoke
 {
     class ConfigurationDataFileList : ConfigurationDataBasicLogin
     {

src/Jackett/Models/IndexerConfig/Bespoke/ConfigurationDataNCore.cs

@@ -7,7 +7,7 @@ using System.Reflection;
 using System.Text;
 using System.Threading.Tasks;
 
-namespace Jackett.Models.IndexerConfig
+namespace Jackett.Models.IndexerConfig.Bespoke
 {
     public class ConfigurationDataNCore : ConfigurationData
     {

src/Jackett/Models/IndexerConfig/Bespoke/ConfigurationDataRuTor.cs

@@ -5,7 +5,7 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 
-namespace Jackett.Models.IndexerConfig
+namespace Jackett.Models.IndexerConfig.Bespoke
 {
     public class ConfigurationDataRuTor : ConfigurationData
     {

src/Jackett/Models/IndexerConfig/Bespoke/ConfigurationDataStrike.cs

@@ -4,7 +4,7 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 
-namespace Jackett.Models.IndexerConfig
+namespace Jackett.Models.IndexerConfig.Bespoke
 {
     public class ConfigurationDataStrike : ConfigurationDataUrl
     {

src/Jackett/Models/IndexerConfig/ConfigurationDataAPIKey.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Jackett.Models.IndexerConfig
+{
+    public class ConfigurationDataAPIKey : ConfigurationData
+    {
+        public ConfigurationData.StringItem Key { get; private set; }
+
+        public ConfigurationDataAPIKey()
+        {
+            Key = new ConfigurationData.StringItem { Name = "APIKey", Value = string.Empty };
+        }
+    }
+}