Implement animetorrents #150

README.md

@@ -20,6 +20,7 @@ We were previously focused on TV but are working on extending searches to allow
 #### Supported Trackers
  * [AlphaRatio](https://alpharatio.cc/)
  * [AnimeBytes](https://animebytes.tv/)
+ * [AnimeTorrents](http://animetorrents.me/)
  * [Avistaz](https://avistaz.to/)
  * [BakaBT](http://bakabt.me/)
  * [bB](http://reddit.com/r/baconbits)

src/Jackett/Indexers/AnimeTorrents.cs

@@ -0,0 +1,176 @@
+using CsQuery;
+using Jackett.Models;
+using Jackett.Services;
+using Jackett.Utils;
+using Jackett.Utils.Clients;
+using Newtonsoft.Json.Linq;
+using NLog;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Text;
+using System.Threading.Tasks;
+using System.Web;
+using Jackett.Models.IndexerConfig;
+using System.Collections.Specialized;
+using System.Globalization;
+
+namespace Jackett.Indexers
+{
+    public class AnimeTorrents : BaseIndexer, IIndexer
+    {
+        private string LoginUrl { get { return SiteLink + "login.php"; } }
+        private string SearchUrl { get { return SiteLink + "ajax/torrents_data.php"; } }
+        private string SearchUrlReferer { get { return SiteLink + "torrents.php?cat=0&searchin=filename&search="; } }
+
+        new ConfigurationDataBasicLogin configData
+        {
+            get { return (ConfigurationDataBasicLogin)base.configData; }
+            set { base.configData = value; }
+        }
+
+        public AnimeTorrents(IIndexerManagerService i, HttpWebClient c, Logger l, IProtectionService ps)
+            : base(name: "AnimeTorrents",
+                description: "Definitive source for anime and manga",
+                link: "http://animetorrents.me/",
+                caps: new TorznabCapabilities(),
+                manager: i,
+                client: c, // Forced HTTP client for custom headers
+                logger: l,
+                p: ps,
+                configData: new ConfigurationDataBasicLogin())
+        {
+            AddCategoryMapping(1, TorznabCatType.MoviesSD); // Anime Movie
+            AddCategoryMapping(6, TorznabCatType.MoviesHD); // Anime Movie HD
+            AddCategoryMapping(2, TorznabCatType.TVAnime); // Anime Series
+            AddCategoryMapping(7, TorznabCatType.TVAnime); // Anime Series HD
+            AddCategoryMapping(5, TorznabCatType.XXXDVD); // Hentai (censored)
+            AddCategoryMapping(9, TorznabCatType.XXXDVD); // Hentai (censored) HD
+            AddCategoryMapping(4, TorznabCatType.XXXDVD); // Hentai (un-censored)
+            AddCategoryMapping(8, TorznabCatType.XXXDVD); // Hentai (un-censored) HD
+            AddCategoryMapping(13, TorznabCatType.BooksForeign); // Light Novel
+            AddCategoryMapping(3, TorznabCatType.BooksComics); // Manga
+            AddCategoryMapping(10, TorznabCatType.BooksComics); // Manga 18+
+            AddCategoryMapping(11, TorznabCatType.TVAnime); // OVA
+            AddCategoryMapping(12, TorznabCatType.TVAnime); // OVA HD
+            AddCategoryMapping(14, TorznabCatType.BooksComics); // Doujin Anime
+            AddCategoryMapping(15, TorznabCatType.XXXDVD); // Doujin Anime 18+
+            AddCategoryMapping(16, TorznabCatType.AudioForeign); // Doujin Music
+            AddCategoryMapping(17, TorznabCatType.BooksComics); // Doujinshi
+            AddCategoryMapping(18, TorznabCatType.BooksComics); // Doujinshi 18+
+            AddCategoryMapping(19, TorznabCatType.Audio); // OST
+        }
+
+        public async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
+        {
+            configData.LoadValuesFromJson(configJson);
+            var pairs = new Dictionary<string, string> {
+                { "username", configData.Username.Value },
+                { "password", configData.Password.Value },
+                { "form", "login" },
+                { "rememberme[]", "1" }
+            };
+
+            var loginPage = await RequestStringWithCookiesAndRetry(LoginUrl, null, null);
+
+            var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, SearchUrl, SiteLink);
+            await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("logout.php"), () =>
+            {
+                CQ dom = result.Content;
+                var errorMessage = dom[".ui-state-error"].Text().Trim();
+                throw new ExceptionWithConfigData(errorMessage, configData);
+            });
+
+            return IndexerConfigurationStatus.RequiresTesting;
+        }
+
+        public async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
+        {
+            var releases = new List<ReleaseInfo>();
+            var searchString = query.GetQueryString();
+            var searchUrl = SearchUrl;
+            var queryCollection = new NameValueCollection();
+
+            queryCollection.Add("total", "146"); // Not sure what this is about but its required!
+
+            var cat = "0";
+            var queryCats = MapTorznabCapsToTrackers(query);
+            if (queryCats.Count == 1)
+            {
+                cat = queryCats.First().ToString();
+            }
+
+            queryCollection.Add("cat", cat);
+            queryCollection.Add("searchin", "filename");
+            queryCollection.Add("search", searchString);
+            queryCollection.Add("page", "1");
+            searchUrl += "?" + queryCollection.GetQueryString();
+
+            var extraHeaders = new Dictionary<string, string>()
+            {
+                { "X-Requested-With", "XMLHttpRequest" }
+            };
+
+            var response = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrlReferer, extraHeaders);
+
+            var results = response.Content;
+            try
+            {
+                CQ dom = results;
+
+                var rows = dom["tr"];
+                foreach (var row in rows.Skip(1))
+                {
+                    var release = new ReleaseInfo();
+                    var qRow = row.Cq();
+                    var qTitleLink = qRow.Find("td:eq(1) a:eq(0)").First();
+                    release.Title = qTitleLink.Find("strong").Text().Trim();
+
+                    // If we search an get no results, we still get a table just with no info.
+                    if (string.IsNullOrWhiteSpace(release.Title))
+                    {
+                        break;
+                    }
+
+                    release.Description = release.Title;
+                    release.Guid = new Uri(qTitleLink.Attr("href"));
+                    release.Comments = release.Guid;
+
+                    var dateString = qRow.Find("td:eq(4)").Text();
+                    release.PublishDate = DateTime.ParseExact(dateString, "dd MMM yy", CultureInfo.InvariantCulture);
+
+                    var qLink = qRow.Find("td:eq(2) a");
+                    release.Link = new Uri(qLink.Attr("href"));
+
+                    var sizeStr = qRow.Find("td:eq(5)").Text();
+                    release.Size = ReleaseInfo.GetBytes(sizeStr);
+
+                    var connections = qRow.Find("td:eq(7)").Text().Trim().Split("/".ToCharArray(),StringSplitOptions.RemoveEmptyEntries);
+
+                    release.Seeders = ParseUtil.CoerceInt(connections[0].Trim());
+                    release.Peers = ParseUtil.CoerceInt(connections[1].Trim()) + release.Seeders;
+
+                    var rCat = row.Cq().Find("td:eq(0) a").First().Attr("href");
+                    var rCatIdx = rCat.IndexOf("cat=");
+                    if (rCatIdx > -1)
+                    {
+                        rCat = rCat.Substring(rCatIdx + 4);
+                    }
+
+                    release.Category = MapTrackerCatToNewznab(rCat);
+
+                    releases.Add(release);
+                }
+            }
+            catch (Exception ex)
+            {
+                OnParseError(results, ex);
+            }
+
+            return releases;
+        }
+    }
+}

src/Jackett/Indexers/BaseIndexer.cs

@@ -246,14 +246,15 @@ namespace Jackett.Indexers
             throw lastException;
         }
 
-        protected async Task<WebClientStringResult> RequestStringWithCookies(string url, string cookieOverride = null, string referer = null)
+        protected async Task<WebClientStringResult> RequestStringWithCookies(string url, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null)
         {
             var request = new Utils.Clients.WebRequest()
             {
                 Url = url,
                 Type = RequestType.GET,
                 Cookies = CookieHeader,
-                Referer = referer
+                Referer = referer,
+                 Headers = headers
             };
 
             if (cookieOverride != null)
@@ -261,14 +262,14 @@ namespace Jackett.Indexers
             return await webclient.GetString(request);
         }
 
-        protected async Task<WebClientStringResult> RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null)
+        protected async Task<WebClientStringResult> RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null, Dictionary<string,string> headers = null)
         {
             Exception lastException = null;
             for (int i = 0; i < 3; i++)
             {
                 try
                 {
-                    return await RequestStringWithCookies(url, cookieOverride, referer);
+                    return await RequestStringWithCookies(url, cookieOverride, referer, headers);
                 }
                 catch (Exception e)
                 {

src/Jackett/Jackett.csproj

@@ -192,6 +192,7 @@
     <Compile Include="Indexers\ImmortalSeed.cs" />
     <Compile Include="Indexers\FileList.cs" />
     <Compile Include="Indexers\Abstract\AvistazTracker.cs" />
+    <Compile Include="Indexers\AnimeTorrents.cs" />
     <Compile Include="Models\ManualSearchResult.cs" />
     <Compile Include="Indexers\Rarbg.cs" />
     <Compile Include="Indexers\TVChaosUK.cs" />
@@ -408,6 +409,9 @@
     <Content Include="Content\logos\animebytes.png">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="Content\logos\animetorrents.png">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="Content\logos\avistaz.png">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Jackett/JackettModule.cs

@@ -21,6 +21,7 @@ namespace Jackett
             var thisAssembly = typeof(JackettModule).Assembly;
             builder.RegisterAssemblyTypes(thisAssembly).Except<IIndexer>().AsImplementedInterfaces().SingleInstance();
             builder.RegisterApiControllers(thisAssembly).InstancePerRequest();
+            builder.RegisterType<HttpWebClient>();
 
             // Register the best web client for the platform or the override
             switch (Startup.ClientOverride)

src/Jackett/Utils/Clients/HttpWebClient.cs

@@ -11,7 +11,7 @@ using System.Threading.Tasks;
 
 namespace Jackett.Utils.Clients
 {
-    class HttpWebClient : IWebClient
+    public class HttpWebClient : IWebClient
     {
         private Logger logger;
 
@@ -70,6 +70,14 @@ namespace Jackett.Utils.Clients
             client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
             HttpResponseMessage response = null;
 
+            if (request.Headers != null)
+            {
+                foreach (var header in request.Headers)
+                {
+                    client.DefaultRequestHeaders.Add(header.Key, header.Value);
+                }
+            }
+
             if (request.Type == RequestType.POST)
             {
                 var content = new FormUrlEncodedContent(request.PostData);

src/Jackett/Utils/Clients/WebRequest.cs

@@ -13,6 +13,7 @@ namespace Jackett.Utils.Clients
         {
             PostData = new List<KeyValuePair<string, string>>();
             Type = RequestType.GET;
+            Headers = new Dictionary<string, string>();
         }
 
         public WebRequest(string url)
@@ -20,6 +21,7 @@ namespace Jackett.Utils.Clients
             PostData = new List<KeyValuePair<string, string>>();
             Type = RequestType.GET;
             Url = url;
+            Headers = new Dictionary<string, string>();
         }
 
         public string Url { get; set; }
@@ -28,6 +30,10 @@ namespace Jackett.Utils.Clients
         public string Referer { get; set; }
         public RequestType Type { get; set; }
 
+        /// <summary>
+        /// Warning this is only implemented on HTTPWebClient currently!
+        /// </summary>
+        public Dictionary<string, string> Headers { get; set; }
 
         public override bool Equals(System.Object obj)
         {