MoreThanTV: Rewrite & Header Support for Login (#11237) resolves #6783

src/Jackett.Common/Indexers/BaseIndexer.cs

@@ -495,7 +495,7 @@ namespace Jackett.Common.Indexers
             return result;
         }
 
-        protected async Task<WebResult> RequestLoginAndFollowRedirect(string url, IEnumerable<KeyValuePair<string, string>> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null, bool accumulateCookies = false)
+        protected async Task<WebResult> RequestLoginAndFollowRedirect(string url, IEnumerable<KeyValuePair<string, string>> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null, bool accumulateCookies = false, Dictionary<string, string> headers = null)
         {
             var request = new WebRequest
             {
@@ -504,7 +504,8 @@ namespace Jackett.Common.Indexers
                 Cookies = cookies,
                 Referer = referer,
                 PostData = data,
-                Encoding = Encoding
+                Encoding = Encoding,
+                Headers = headers,
             };
             var response = await webclient.GetResultAsync(request);
             CheckSiteDown(response);

src/Jackett.Common/Indexers/MoreThanTV.cs

@@ -8,6 +8,7 @@ using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using AngleSharp.Dom;
+using AngleSharp.Html.Dom;
 using AngleSharp.Html.Parser;
 using Jackett.Common.Models;
 using Jackett.Common.Models.IndexerConfig;
@@ -15,10 +16,13 @@ using Jackett.Common.Services.Interfaces;
 using Jackett.Common.Utils;
 using Jackett.Common.Utils.Clients;
 using Newtonsoft.Json.Linq;
+using System.Web;
 using NLog;
+using static Jackett.Common.Models.IndexerConfig.ConfigurationData;
 
 namespace Jackett.Common.Indexers
 {
+    // This tracker uses a hybrid Luminance (based on GazelleTracker) 
     [ExcludeFromCodeCoverage]
     public class MoreThanTV : BaseWebIndexer
     {
@@ -26,13 +30,17 @@ namespace Jackett.Common.Indexers
             "https://www.morethan.tv/"
         };
 
-        private string LoginUrl => SiteLink + "login.php";
+        private string LoginUrl => SiteLink + "login";
-        private string SearchUrl => SiteLink + "ajax.php?action=browse&searchstr=";
+        private string BrowseUrl => SiteLink + "torrents.php";
-        private string DownloadUrl => SiteLink + "torrents.php?action=download&id=";
+        private string DetailsUrl => SiteLink + "details.php";
-        private string DetailsUrl => SiteLink + "torrents.php?torrentid=";
+
+        private string _sort;
+        private string _order;
 
         private ConfigurationDataBasicLogin ConfigData => (ConfigurationDataBasicLogin)configData;
 
+        private readonly Dictionary<string, string> _emulatedBrowserHeaders = new Dictionary<string, string>();
+
         public MoreThanTV(IIndexerConfigurationService configService, WebClient c, Logger l, IProtectionService ps,
             ICacheService cs)
             : base(id: "morethantv",
@@ -43,11 +51,11 @@ namespace Jackett.Common.Indexers
                    {
                        TvSearchParams = new List<TvSearchParam>
                        {
-                           TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep
+                           TvSearchParam.Q
                        },
                        MovieSearchParams = new List<MovieSearchParam>
                        {
-                           MovieSearchParam.Q, MovieSearchParam.ImdbId
+                           MovieSearchParam.Q
                        }
                    },
                    configService: configService,
@@ -61,35 +69,112 @@ namespace Jackett.Common.Indexers
             Language = "en-us";
             Type = "private";
 
+            var sort = new SelectItem(new Dictionary<string, string>
+            {
+                {"time", "time"},
+                {"size", "size"},
+                {"snatched", "snatched"},
+                {"seeders", "seeders"},
+                {"leechers", "leechers"},
+            })
+            { Name = "Sort requested from site", Value = "time" };
+            configData.AddDynamic("sort", sort);
+
+            var order = new SelectItem(new Dictionary<string, string>
+            {
+                {"desc", "desc"},
+                {"asc", "asc"}
+            })
+            { Name = "Order requested from site", Value = "desc" };
+            configData.AddDynamic("order", order);
+
             AddCategoryMapping(1, TorznabCatType.Movies);
             AddCategoryMapping(2, TorznabCatType.TV);
             AddCategoryMapping(3, TorznabCatType.Other);
         }
 
+        /// <summary>
+        /// Parse and Return CSRF token
+        /// </summary>
+        /// <param name="content"></param>
+        /// <returns></returns>
+        private string GetToken(string content)
+        {
+            try
+            {
+                var parser = new HtmlParser();
+                var dom = parser.ParseDocument(content.Trim());
+                return dom.QuerySelector<IHtmlInputElement>("input[name=\"token\"]").Value;
+            }
+            catch (Exception e)
+            {
+                throw new Exception("Token Could not be parsed from Response, Error?", e);
+            }
+        }
+
+        /// <summary>
+        /// Emulate browser headers -- REQUIRED
+        /// </summary>
+        private void SetRequestHeaders()
+        {
+            _emulatedBrowserHeaders.Clear();
+
+            _emulatedBrowserHeaders.Add("referer", SiteLink);
+            _emulatedBrowserHeaders.Add("Upgrade-Insecure-Requests", "1");
+            _emulatedBrowserHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.72 Safari/537.36");
+        }
+
         public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
         {
             LoadValuesFromJson(configJson);
+
             var pairs = new Dictionary<string, string> {
                 { "username", ConfigData.Username.Value },
                 { "password", ConfigData.Password.Value },
-                { "login", "Log in" },
+                { "submit", "login" },
-                { "keeplogged", "1" }
+                { "keeplogged", "1" },
+                { "cinfo", "3440|1440|24|360" }
             };
-            var preRequest = await RequestWithCookiesAndRetryAsync(LoginUrl, string.Empty);
-            var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, preRequest.Cookies, true, SearchUrl, SiteLink);
 
-            await ConfigureIfOK(result.Cookies, result.ContentString != null && result.ContentString.Contains("status\":\"success\""), () =>
+            SetRequestHeaders();
+
+            // Fetch CSRF token
+            var preRequest = await RequestWithCookiesAndRetryAsync(LoginUrl, referer: SiteLink, headers: _emulatedBrowserHeaders);
+            // Check if user is logged in. /login redirects to / if so)
+            if (preRequest.IsRedirect)
             {
-                if (result.ContentString.Contains("Your IP address has been banned."))
+                await FollowIfRedirect(preRequest, SiteLink, null, preRequest.Cookies, true);
-                    throw new ExceptionWithConfigData("Your IP address has been banned.", ConfigData);
+            }
 
+            // sid was not set after redirect, attempt to log in again
+            if (!preRequest.Cookies.Contains("sid="))
+            {
+                string token = null;
+
+                try
+                {
+                    token = GetToken(preRequest.ContentString);
+                }
+                catch (Exception)
+                {
+                    string errorMessage = ParseErrorMessage(preRequest);
+                    throw new ExceptionWithConfigData(errorMessage, configData);
+                }
+
+                // Add CSRF Token to payload
+                pairs.Add("token", token);
+
+                var response = await RequestLoginAndFollowRedirect(LoginUrl, pairs, preRequest.Cookies, true, null, SiteLink, headers: _emulatedBrowserHeaders);
+
+                await ConfigureIfOK(response.Cookies, response.Cookies.Contains("sid="), () =>
+                {
+                // Couldn't find "sid" cookie, so check for error
                 var parser = new HtmlParser();
-                var dom = parser.ParseDocument(result.ContentString);
+                    var dom = parser.ParseDocument(response.ContentString);
-                foreach (var element in dom.QuerySelectorAll("#loginform > table"))
+                    var errorMessage = dom.QuerySelector(".flash.error").TextContent.Trim();
-                    element.Remove();
+                    throw new ExceptionWithConfigData(errorMessage, configData);
-                var errorMessage = dom.QuerySelector("#loginform").TextContent.Trim().Replace("\n\t", " ");
-                throw new ExceptionWithConfigData(errorMessage, ConfigData);
                 });
+            }
 
             return IndexerConfigurationStatus.RequiresTesting;
         }
@@ -98,44 +183,45 @@ namespace Jackett.Common.Indexers
         {
             var releases = new List<ReleaseInfo>();
 
-            if (!string.IsNullOrWhiteSpace(query.ImdbID))
-                await GetReleases(releases, query, query.GetQueryString());
-            else
-            {
             var searchQuery = query.GetQueryString();
             searchQuery = searchQuery.Replace("Marvels", "Marvel"); // strip 's for better results
             var newSearchQuery = Regex.Replace(searchQuery, @"(S\d{2})$", "$1*"); // If we're just seaching for a season (no episode) append an * to include all episodes of that season.
-                await GetReleases(releases, query, newSearchQuery);
+            await GetReleasesAsync(releases, query, newSearchQuery);
 
             // Always search for torrent groups (complete seasons) too
             var seasonMatch = new Regex(@".*\s[Ss]{1}\d{2}([Ee]{1}\d{2,3})?$").Match(searchQuery);
             if (seasonMatch.Success)
             {
                 newSearchQuery = Regex.Replace(searchQuery, @"[Ss]{1}\d{2}([Ee]{1}\d{2,3})?", $"Season {query.Season}");
-                    await GetReleases(releases, query, newSearchQuery);
+                await GetReleasesAsync(releases, query, newSearchQuery);
-                }
             }
 
             return releases;
         }
 
+        public override void LoadValuesFromJson(JToken jsonConfig, bool useProtectionService = false)
+        {
+            base.LoadValuesFromJson(jsonConfig, useProtectionService);
+
+            var sort = (SelectItem)configData.GetDynamic("sort");
+            _sort = sort != null ? sort.Value : "time";
+
+            var order = (SelectItem)configData.GetDynamic("order");
+            _order = order != null && order.Value.Equals("asc") ? order.Value : "desc";
+        }
+
         private string GetTorrentSearchUrl(TorznabQuery query, string searchQuery)
         {
             var qc = new NameValueCollection
             {
-                { "tags_type", "1" },
+                { "order_by", _sort },
-                { "order_by", "time" },
+                { "order_way",  _order },
-                { "order_way", "desc" },
+                { "action", "advanced" },
-                { "group_results", "1" },
+                { "sizetype", "gb" },
-                { "action", "basic" },
+                { "sizerange", "0.01" },
-                { "searchsubmit", "1" }
+                { "title", searchQuery }
             };
 
-            if (!string.IsNullOrWhiteSpace(query.ImdbID))
-                qc.Add("description", query.ImdbID);
-            else
-                qc.Add("searchstr", searchQuery);
-
             if (query.Categories.Contains(TorznabCatType.Movies.ID))
                 qc.Add("filter_cat[1]", "1");
             if (query.Categories.Contains(TorznabCatType.TV.ID))
@@ -143,16 +229,16 @@ namespace Jackett.Common.Indexers
             if (query.Categories.Contains(TorznabCatType.Other.ID))
                 qc.Add("filter_cat[3]", "1");
 
-            return SiteLink + "torrents.php?" + qc.GetQueryString();
+            return BrowseUrl + "?" + qc.GetQueryString();
         }
 
-        private async Task GetReleases(ICollection<ReleaseInfo> releases, TorznabQuery query, string searchQuery)
+        private async Task GetReleasesAsync(ICollection<ReleaseInfo> releases, TorznabQuery query, string searchQuery)
         {
             var searchUrl = GetTorrentSearchUrl(query, searchQuery);
             var response = await RequestWithCookiesAndRetryAsync(searchUrl);
             if (response.IsRedirect)
             {
-                // re login
+                // re-login
                 await ApplyConfiguration(null);
                 response = await RequestWithCookiesAndRetryAsync(searchUrl);
             }
@@ -161,99 +247,31 @@ namespace Jackett.Common.Indexers
             {
                 var parser = new HtmlParser();
                 var document = parser.ParseDocument(response.ContentString);
-                var groups = document.QuerySelectorAll(".torrent_table > tbody > tr.group");
+                var torrents = document.QuerySelectorAll("#torrent_table > tbody > tr.torrent");
-                var torrents = document.QuerySelectorAll(".torrent_table > tbody > tr.torrent");
+                var movies = new[] { "movie" };
+                var tv = new[] { "season", "episode" };
 
-                // Loop through all torrent (season) groups
+                // Loop through all torrents checking for groups
-                foreach (var group in groups)
-                {
-                    var showName = group.QuerySelector(".tp-showname a").InnerHtml.Replace("(", "").Replace(")", "").Replace(' ', '.');
-                    var season = group.QuerySelector(".big_info a").InnerHtml;
-                    var seasonNumber = SeasonToNumber(season);
-                    if (seasonNumber != null && query.Season > 0 && seasonNumber != query.Season) // filter unwanted seasons
-                        continue;
-                    var seasonTag = SeasonNumberToShortSeason(seasonNumber) ?? season;
-
-                    // Loop through all group items
-                    var previousElement = group;
-                    var qualityEdition = string.Empty;
-                    while (true)
-                    {
-                        var groupItem = previousElement.NextElementSibling;
-
-                        if (groupItem == null)
-                            break;
-
-                        if (!groupItem.ClassList[0].Equals("group_torrent") ||
-                            !groupItem.ClassList[1].StartsWith("groupid_"))
-                            break;
-
-                        // Found a new edition
-                        if (groupItem.ClassList[2].Equals("edition"))
-                            qualityEdition = groupItem.QuerySelector(".edition_info strong").TextContent.Split('/')[1].Trim();
-                        else if (groupItem.ClassList[2].StartsWith("edition_"))
-                        {
-                            if (qualityEdition.Equals(string.Empty))
-                                break;
-
-                            // Parse required data
-                            var downloadAnchor = groupItem.QuerySelectorAll("td a").Last();
-                            var qualityData = downloadAnchor.InnerHtml.Split('/');
-
-                            switch (qualityData.Length)
-                            {
-                                case 0:
-                                    Array.Resize(ref qualityData, 2);
-                                    qualityData[0] = " ";
-                                    qualityData[1] = " ";
-                                    break;
-                                case 1:
-                                    Array.Resize(ref qualityData, 2);
-                                    qualityData[1] = " ";
-                                    break;
-                            }
-
-                            // Replace 4K quality tag with 2160p, so Sonarr etc. can properly parse it
-                            qualityData[1] = qualityData[1].Replace("4K", "2160p");
-
-                            // Build title
-                            var title = string.Join(".", new List<string>
-                            {
-                                showName,
-                                seasonTag,
-                                qualityData[1].Trim(),
-                                qualityEdition, // Audio quality should be after this one. Unobtainable at the moment.
-                                $"{qualityData[0].Trim()}-MTV"
-                            });
-
-                            releases.Add(GetReleaseInfo(groupItem, downloadAnchor, title, TorznabCatType.TV.ID));
-                        }
-                        else
-                            break;
-
-                        previousElement = groupItem;
-                    }
-                }
-
-                // Loop through all torrents
                 foreach (var torrent in torrents)
                 {
                     // Parse required data
-                    var downloadAnchor = torrent.QuerySelector(".big_info > .group_info > a");
+                    var torrentGroup = torrent.QuerySelectorAll("table a[href^=\"/torrents.php?action=download\"]");
-                    var title = downloadAnchor.TextContent;
+                    foreach (var downloadAnchor in torrentGroup)
+                    {
+                        var title = downloadAnchor.ParentElement.ParentElement.ParentElement.TextContent.Trim();
+                        title = CleanUpTitle(title);
 
-                    int category;
+                        var category = torrent.QuerySelector(".cats_col div").GetAttribute("title");
-                    var categories = torrent.QuerySelector(".cats_col div").ClassList;
+                        // default to Other
-                    if (categories.Contains("cats_tv"))
+                        int categoryId = TorznabCatType.Other.ID;
-                        category = TorznabCatType.TV.ID;
-                    else if (categories.Contains("cats_movies"))
-                        category = TorznabCatType.Movies.ID;
-                    else if (categories.Contains("cats_other"))
-                        category = TorznabCatType.Other.ID;
-                    else
-                        throw new Exception("Couldn't find category.");
 
-                    releases.Add(GetReleaseInfo(torrent, downloadAnchor, title, category));
+                        if (movies.Any(category.Contains))
+                            categoryId = TorznabCatType.Movies.ID;
+                        else if (tv.Any(category.Contains))
+                            categoryId = TorznabCatType.TV.ID;
+
+                        releases.Add(GetReleaseInfo(torrent, downloadAnchor, title, categoryId));
+                    }
                 }
             }
             catch (Exception ex)
@@ -262,47 +280,73 @@ namespace Jackett.Common.Indexers
             }
         }
 
+        /// <summary>
+        /// Gather Release info from torrent table. Target using css
+        /// </summary>
+        /// <param name="row"></param>
+        /// <param name="downloadAnchor"></param>
+        /// <param name="title"></param>
+        /// <param name="category"></param>
+        /// <returns></returns>
         private ReleaseInfo GetReleaseInfo(IElement row, IElement downloadAnchor, string title, int category)
         {
-            var downloadAnchorHref = downloadAnchor.Attributes["href"].Value;
-            var torrentId = downloadAnchorHref.Substring(downloadAnchorHref.LastIndexOf('=') + 1);
-            if (torrentId.Contains('#'))
-                torrentId = torrentId.Split('#')[0];
 
-            var qFiles = row.QuerySelector("td:nth-last-child(6)");
+            // count from bottom
-            var files = ParseUtil.CoerceLong(qFiles.TextContent);
+            const int FILES_COL = 8;
-            var qPublishDate = row.QuerySelector(".time.tooltip").Attributes["title"].Value;
+            /*const int COMMENTS_COL = 7;*/
+            const int DATE_COL = 6;
+            const int FILESIZE_COL = 5;
+            const int SNATCHED_COL = 4;
+            const int SEEDS_COL = 3;
+            const int LEECHERS_COL = 2;
+            /*const int USER_COL = 1;*/
+
+
+            string downloadAnchorHref = (downloadAnchor as IHtmlAnchorElement).Href;
+            var queryParams = HttpUtility.ParseQueryString(downloadAnchorHref, Encoding.UTF8);
+            string torrentId = queryParams["id"];
+
+            var qFiles = row.QuerySelector("td:nth-last-child(" + FILES_COL + ")").TextContent;
+
+            var fileCount = ParseUtil.CoerceLong(qFiles);
+            var qPublishDate = row.QuerySelector("td:nth-last-child(" + DATE_COL + ") .time").Attributes["title"].Value;
             var publishDate = DateTime.ParseExact(qPublishDate, "MMM dd yyyy, HH:mm", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).ToLocalTime();
             var qPoster = row.QuerySelector("div.tp-banner img")?.GetAttribute("src");
-            var poster = (qPoster != null && !qPoster.Contains("/static/styles/")) ? new Uri(qPoster) : null;
+            var poster = (qPoster != null && !qPoster.Contains("caticons")) ? new Uri(qPoster) : null;
             var description = row.QuerySelector("div.tags")?.TextContent.Trim();
+            var fileSize = row.QuerySelector("td:nth-last-child(" + FILESIZE_COL + ")").TextContent.Trim();
+            var snatched = row.QuerySelector("td:nth-last-child(" + SNATCHED_COL + ")").TextContent.Trim();
+            var seeds = row.QuerySelector("td:nth-last-child(" + SEEDS_COL + ")").TextContent.Trim();
+            var leechs = row.QuerySelector("td:nth-last-child(" + LEECHERS_COL + ")").TextContent.Trim();
 
-            var torrentData = row.QuerySelectorAll(".number_column");
+            if (fileSize.Length <= 0 || snatched.Length <= 0 || seeds.Length <= 0 || leechs.Length <= 0)
-            if (torrentData.Length != 4) // Size (xx.xx GB[ (Max)]) Snatches (xx) Seeders (xx) Leechers (xx)
+            {
-                throw new Exception($"We expected 4 torrent datas, instead we have {torrentData.Length}.");
+                // Size (xx.xx GB[ (Max)]) Snatches (xx) Seeders (xx) Leechers (xx)
+                throw new Exception($"We expected 4 torrent datas.");
+            }
 
-            var size = ReleaseInfo.GetBytes(torrentData[0].TextContent);
+            long size = ReleaseInfo.GetBytes(fileSize);
-            var grabs = int.Parse(torrentData[1].TextContent, NumberStyles.AllowThousands, CultureInfo.InvariantCulture);
+            int grabs = int.Parse(snatched, NumberStyles.AllowThousands, CultureInfo.InvariantCulture);
-            var seeders = int.Parse(torrentData[2].TextContent, NumberStyles.AllowThousands, CultureInfo.InvariantCulture);
+            int seeders = int.Parse(seeds, NumberStyles.AllowThousands, CultureInfo.InvariantCulture);
-            var leechers = int.Parse(torrentData[3].TextContent, NumberStyles.AllowThousands, CultureInfo.InvariantCulture);
+            int leechers = int.Parse(leechs, NumberStyles.AllowThousands, CultureInfo.InvariantCulture);
-            var details = new Uri(DetailsUrl + torrentId);
+            var detailsUri = new Uri(DetailsUrl + "?torrentid=" + torrentId);
-            var link = new Uri(DownloadUrl + torrentId);
+            var downloadLink = new Uri(BrowseUrl + "?action=download&id=" + torrentId);
 
             return new ReleaseInfo
             {
                 Title = title,
-                Category = new List<int> { category }, // Who seasons movies right
+                Category = new List<int> { category },
-                Link = link,
+                Link = downloadLink,
                 PublishDate = publishDate,
                 Poster = poster,
                 Description = description,
                 Seeders = seeders,
                 Peers = seeders + leechers,
-                Files = files,
+                Files = fileCount,
                 Size = size,
                 Grabs = grabs,
-                Guid = details,
+                Guid = downloadLink,
-                Details = details,
+                Details = detailsUri,
                 DownloadVolumeFactor = 0, // ratioless tracker
                 UploadVolumeFactor = 1
             };
@@ -325,5 +369,41 @@ namespace Jackett.Common.Indexers
                 return null;
             return $"S{season:00}";
         }
+
+        /// <summary>
+        /// Parse Error Messages from using CSS classes
+        /// </summary>
+        /// <param name="response"></param>
+        /// <returns></returns>
+        private string ParseErrorMessage(WebResult response)
+        {
+            var parser = new HtmlParser();
+            var dom = parser.ParseDocument(response.ContentString);
+            var errorMessage = "Unknown Error";
+
+            switch (response.Status)
+            {
+                case System.Net.HttpStatusCode.Forbidden:
+                    errorMessage = dom.QuerySelector(".time").Parent.TextContent.Trim();
+                    break;
+                default:
+                    errorMessage = dom.QuerySelector(".flash.error").TextContent.Trim();
+                    break;
+
+            }
+            return errorMessage;
+        }
+
+        /// <summary>
+        /// Clean Up any title stuff
+        /// </summary>
+        /// <param name="title"></param>
+        /// <returns></returns>
+        private string CleanUpTitle(string title)
+        {
+            return title
+                .Replace(".", " ")
+                .Replace("4K", "2160p"); // sonarr cleanup
+        }
     }
 }