Fix parse errors in Fuzer and Hebits (#7506) resolves #7502

src/Jackett.Common/Indexers/Fuzer.cs

@@ -6,13 +6,13 @@ using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
-using AngleSharp.Dom;
 using AngleSharp.Html.Parser;
 using Jackett.Common.Helpers;
 using Jackett.Common.Models;
 using Jackett.Common.Models.IndexerConfig;
 using Jackett.Common.Services.Interfaces;
 using Jackett.Common.Utils;
+using Jackett.Common.Utils.Clients;
 using Newtonsoft.Json.Linq;
 using NLog;
 
@@ -20,13 +20,13 @@ namespace Jackett.Common.Indexers
 {
     public class Fuzer : BaseWebIndexer
     {
-        public override string[] LegacySiteLinks { get; protected set; } = new string[] {
+        public override string[] LegacySiteLinks { get; protected set; } =
-            "https://fuzer.me/",
+        {
+            "https://fuzer.me/"
         };
 
         private string SearchUrl => SiteLink + "browse.php";
         private string LoginUrl => SiteLink + "login.php";
-        private const int MAXPAGES = 3;
 
         private new ConfigurationDataRecaptchaLogin configData
         {
@@ -130,18 +130,14 @@ namespace Jackett.Common.Indexers
         public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
         {
             LoadValuesFromJson(configJson);
-
             if (!string.IsNullOrWhiteSpace(configData.Captcha.Cookie))
             {
                 CookieHeader = configData.Captcha.Cookie;
                 try
                 {
                     var results = await PerformQuery(new TorznabQuery());
-                    if (results.Count() == 0)
+                    if (!results.Any())
-                    {
                         throw new Exception("Your cookie did not work");
-                    }
-
                     IsConfigured = true;
                     SaveConfig();
                     return IndexerConfigurationStatus.Completed;
@@ -154,8 +150,8 @@ namespace Jackett.Common.Indexers
             }
 
             var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
-
+            var pairs = new Dictionary<string, string>
-            var pairs = new Dictionary<string, string> {
+            {
                 {"vb_login_username", configData.Username.Value},
                 {"vb_login_password", ""},
                 {"securitytoken", "guest"},
@@ -164,57 +160,38 @@ namespace Jackett.Common.Indexers
                 {"vb_login_md5password_utf", StringUtil.Hash(configData.Password.Value).ToLower()},
                 {"cookieuser", "1"}
             };
-
             var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, null, LoginUrl);
-
+            await ConfigureIfOK(result.Cookies, result.Content?.Contains("images/loading.gif") == true,
-            await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("images/loading.gif"), () =>
+                                () => throw new ExceptionWithConfigData("Couldn't login", configData));
-            {
-                var errorMessage = "Couldn't login";
-                throw new ExceptionWithConfigData(errorMessage, configData);
-            });
             Thread.Sleep(2);
             return IndexerConfigurationStatus.RequiresTesting;
         }
 
         protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
         {
-            var results = await performRegularQuery(query);
+            var results = await PerformRegularQueryAsync(query);
-            if (results.Count() == 0 && !query.IsImdbQuery)
+            if (!results.Any() && !query.IsImdbQuery)
-            {
+                return await PerformHebrewQueryAsync(query);
-                return await performHebrewQuery(query);
-            }
-
             return results;
         }
 
-        private async Task<IEnumerable<ReleaseInfo>> performHebrewQuery(TorznabQuery query)
+        private async Task<IEnumerable<ReleaseInfo>> PerformHebrewQueryAsync(TorznabQuery query)
         {
-            var name = await getHebName(query.SearchTerm);
+            var name = await GetHebNameAsync(query.SearchTerm);
-
             if (string.IsNullOrEmpty(name))
-            {
                 return new List<ReleaseInfo>();
-            }
+            return await PerformRegularQueryAsync(query, name);
-            else
-            {
-                return await performRegularQuery(query, name);
-            }
         }
 
-        private async Task<IEnumerable<ReleaseInfo>> performRegularQuery(TorznabQuery query, string hebName = null)
+        private async Task<IEnumerable<ReleaseInfo>> PerformRegularQueryAsync(TorznabQuery query, string hebName = null)
         {
             var releases = new List<ReleaseInfo>();
-            var searchurls = new List<string>();
             var searchUrl = SearchUrl;
-            var queryCollection = new NameValueCollection();
             var searchString = query.GetQueryString();
             if (query.IsImdbQuery)
                 searchString = query.ImdbID;
-
             if (hebName != null)
-            {
                 searchString = hebName + " - עונה " + query.Season + " פרק " + query.Episode;
-            }
             searchUrl += "?";
             if (!string.IsNullOrWhiteSpace(searchString))
             {
@@ -222,11 +199,7 @@ namespace Jackett.Common.Indexers
                 searchUrl += "&query=" + strEncoded + "&matchquery=any";
             }
 
-            foreach (var cat in MapTorznabCapsToTrackers(query))
+            searchUrl = MapTorznabCapsToTrackers(query).Aggregate(searchUrl, (current, cat) => $"{current}&c[]={cat}");
-            {
-                searchUrl += "&c[]=" + cat;
-            }
-
             var data = await RequestStringWithCookiesAndRetry(searchUrl);
             try
             {
@@ -236,63 +209,43 @@ namespace Jackett.Common.Indexers
                 foreach (var row in rows)
                 {
                     var release = new ReleaseInfo();
-                    var main_title_link = row.QuerySelector("div.main_title > a");
+                    var mainTitleLink = row.QuerySelector("div.main_title > a");
-                    release.Title = main_title_link.GetAttribute("longtitle");
+                    release.Title = mainTitleLink.GetAttribute("longtitle");
-                    if (release.Title.IsNullOrEmptyOrWhitespace())
+                    if (string.IsNullOrWhiteSpace(release.Title))
-                        release.Title = main_title_link.TextContent;
+                        release.Title = mainTitleLink.TextContent;
-
                     release.MinimumRatio = 1;
                     release.MinimumSeedTime = 172800; // 48 hours
 
-                    if (ParseUtil.TryCoerceInt(row.QuerySelector("td:nth-child(7) > div").TextContent, out var seeders))
-                    {
-                        release.Seeders = seeders;
-                        if (ParseUtil.TryCoerceInt(row.QuerySelector("td:nth-child(8) > div").TextContent, out var peers))
-                        {
-                            release.Peers = peers + release.Seeders;
-                        }
-                    }
                     release.Grabs = ParseUtil.CoerceLong(row.QuerySelector("td:nth-child(5)").TextContent.Replace(",", ""));
                     release.Seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-child(6)").TextContent.Replace(",", ""));
-                    release.Peers = ParseUtil.CoerceInt(row.QuerySelector("td:nth-child(7)").TextContent.Replace(",", "")) + release.Seeders;
+                    release.Peers = ParseUtil.CoerceInt(row.QuerySelector("td:nth-child(7)").TextContent.Replace(",", "")) +
+                                    release.Seeders;
                     var fullSize = row.QuerySelector("td:nth-child(4)").TextContent;
                     release.Size = ReleaseInfo.GetBytes(fullSize);
-
                     release.Comments = new Uri(SiteLink + row.QuerySelector("a.threadlink[href]").GetAttribute("href"));
                     release.Link = new Uri(SiteLink + row.QuerySelector("a:has(div.dlimg)").GetAttribute("href"));
                     release.Guid = release.Comments;
-                    try
+                    //some releases have invalid banner URLs, ignore the banners in this case
-                    {
+                    if (Uri.TryCreate(row.QuerySelector("a[imgsrc]").GetAttribute("imgsrc"),
-                        release.BannerUrl = new Uri(row.QuerySelector("a[imgsrc]").GetAttribute("imgsrc"));
+                                      UriKind.Absolute, out var banner))
-                    }
+                        release.BannerUrl = banner;
-                    catch (Exception)
-                    {
-                        // do nothing, some releases have invalid banner URLs, ignore the banners in this case
-                    }
-
                     var dateStringAll = row.QuerySelector("div.up_info2").ChildNodes.Last().ToString();
                     var dateParts = dateStringAll.Split(' ');
                     var dateString = dateParts[dateParts.Length - 2] + " " + dateParts[dateParts.Length - 1];
                     release.PublishDate = DateTime.ParseExact(dateString, "dd/MM/yy HH:mm", CultureInfo.InvariantCulture);
-
                     var categoryLink = row.QuerySelector("a[href^=\"/browse.php?cat=\"]").GetAttribute("href");
                     var catid = ParseUtil.GetArgumentFromQueryString(categoryLink, "cat");
                     release.Category = MapTrackerCatToNewznab(catid);
-
                     if (row.QuerySelector("a[href^=\"?freeleech=1\"]") != null)
                         release.DownloadVolumeFactor = 0;
                     else
                         release.DownloadVolumeFactor = 1;
-
                     release.UploadVolumeFactor = 1;
-
+                    var subTitle = row.QuerySelector("div.sub_title");
-                    var sub_title = row.QuerySelector("div.sub_title");
+                    var imdbLink = subTitle.QuerySelector("span.imdb-inline > a");
-                    var imdb_link = sub_title.QuerySelector("span.imdb-inline > a");
+                    if (imdbLink != null)
-                    release.Imdb = ParseUtil.GetLongFromString(imdb_link.GetAttribute("href"));
+                        release.Imdb = ParseUtil.GetLongFromString(imdbLink.GetAttribute("href"));
-                    foreach(var element in sub_title.QuerySelectorAll("span.imdb-inline"))
+                    release.Description = subTitle.FirstChild.TextContent;
-                        element.Remove();
-                    release.Description = sub_title.TextContent;
-
                     releases.Add(release);
                 }
             }
@@ -304,37 +257,35 @@ namespace Jackett.Common.Indexers
             return releases;
         }
 
-        private async Task<string> getHebName(string searchTerm)
+        private async Task<string> GetHebNameAsync(string searchTerm)
         {
-            const string site = "http://thetvdb.com";
+            var queryString = new NameValueCollection
-            var url = site + "/index.php?searchseriesid=&tab=listseries&function=Search&";
+            {
-            url += "string=" + searchTerm; // eretz + nehedert
+                {"searchseriesid", ""},
-
+                {"tab", "listseries" },
-            var results = await RequestStringWithCookies(url);
+                { "function", "Search"},
-
+                {"string", searchTerm }, // eretz + nehedert
+            };
+            var site = new UriBuilder
+            {
+                Scheme = "http",
+                Host = "thetvdb.com",
+                Path = "index.php",
+                Query = queryString.GetQueryString()
+            };
+            var results = await RequestStringWithCookies(site.ToString());
             var parser = new HtmlParser();
             var dom = parser.ParseDocument(results.Content);
-
-            var rowCount = 0;
             var rows = dom.QuerySelectorAll("#listtable > tbody > tr");
-
+            foreach (var row in rows.Skip(1))
-            foreach (var row in rows)
             {
-                if (rowCount < 1)
-                {
-                    rowCount++;
-                    continue;
-                }
 
                 var link = row.QuerySelector("td:nth-child(1) > a");
-                if (link.TextContent.Trim().ToLower() == searchTerm.Trim().ToLower())
+                if (string.Equals(link.TextContent.Trim(), searchTerm.Trim(), StringComparison.CurrentCultureIgnoreCase))
                 {
                     var address = link.GetAttribute("href");
                     if (string.IsNullOrEmpty(address))
-                    { continue; }
+                        continue;
-
-                    var realAddress = site + address.Replace("lid=7", "lid=24");
-                    var realData = await RequestStringWithCookies(realAddress);
                     var realDom = parser.ParseDocument(results.Content);
                     return realDom.QuerySelector("#content:nth-child(1) > h1").TextContent;
                 }

src/Jackett.Common/Indexers/Hebits.cs

@@ -1,10 +1,10 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
+using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
-using AngleSharp.Dom;
 using AngleSharp.Html.Parser;
 using Jackett.Common.Helpers;
 using Jackett.Common.Models;
@@ -18,7 +18,6 @@ namespace Jackett.Common.Indexers
 {
     public class Hebits : BaseWebIndexer
     {
-        private string LoginUrl => SiteLink + "login.php";
         private string LoginPostUrl => SiteLink + "takeloginAjax.php";
         private string SearchUrl => SiteLink + "browse.php?sort=4&type=desc";
 
@@ -43,13 +42,11 @@ namespace Jackett.Common.Indexers
             Encoding = Encoding.GetEncoding("windows-1255");
             Language = "he-il";
             Type = "private";
-
             AddCategoryMapping(19, TorznabCatType.MoviesSD);
             AddCategoryMapping(25, TorznabCatType.MoviesOther); // Israeli Content
             AddCategoryMapping(20, TorznabCatType.MoviesDVD);
             AddCategoryMapping(36, TorznabCatType.MoviesBluRay);
             AddCategoryMapping(27, TorznabCatType.MoviesHD);
-
             AddCategoryMapping(7, TorznabCatType.TVSD); // Israeli SDTV
             AddCategoryMapping(24, TorznabCatType.TVSD); // English SDTV
             AddCategoryMapping(1, TorznabCatType.TVHD); // Israel HDTV
@@ -59,7 +56,8 @@ namespace Jackett.Common.Indexers
         public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
         {
             LoadValuesFromJson(configJson);
-            var pairs = new Dictionary<string, string> {
+            var pairs = new Dictionary<string, string>
+            {
                 {"username", configData.Username.Value},
                 {"password", configData.Password.Value}
             };
@@ -67,7 +65,8 @@ namespace Jackett.Common.Indexers
             // Get inital cookies
             CookieHeader = string.Empty;
             var result = await RequestLoginAndFollowRedirect(LoginPostUrl, pairs, CookieHeader, true, null, SiteLink);
-            await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("OK"), () =>
+            await ConfigureIfOK(
+                result.Cookies, result.Content?.Contains("OK") == true, () =>
                 {
                     var parser = new HtmlParser();
                     var dom = parser.ParseDocument(result.Content);
@@ -83,82 +82,48 @@ namespace Jackett.Common.Indexers
             var releases = new List<ReleaseInfo>();
             var searchString = query.GetQueryString();
             var searchUrl = SearchUrl;
-
             if (!string.IsNullOrWhiteSpace(searchString))
-            {
                 searchUrl += "&search=" + WebUtilityHelpers.UrlEncode(searchString, Encoding);
-            }
-            string.Format(SearchUrl, WebUtilityHelpers.UrlEncode(searchString, Encoding));
-
             var cats = MapTorznabCapsToTrackers(query);
             if (cats.Count > 0)
-            {
+                searchUrl = cats.Aggregate(searchUrl, (url, cat) => $"{url}&c{cat}=1");
-                foreach (var cat in cats)
-                {
-                    searchUrl += "&c" + cat + "=1";
-                }
-            }
-
             var response = await RequestStringWithCookies(searchUrl);
             try
             {
                 var parser = new HtmlParser();
                 var dom = parser.ParseDocument(response.Content);
-
                 var rows = dom.QuerySelectorAll(".browse > div > div");
-
                 foreach (var row in rows)
                 {
                     var release = new ReleaseInfo();
-
-                    var debug = row.InnerHtml;
-
                     release.MinimumRatio = 1;
                     release.MinimumSeedTime = 172800; // 48 hours
-
                     var qTitle = row.QuerySelector(".bTitle");
                     var titleParts = qTitle.TextContent.Split('/');
-                    if (titleParts.Length >= 2)
+                    release.Title = titleParts.Length >= 2 ? titleParts[1].Trim() : titleParts[0].Trim();
-                        release.Title = titleParts[1].Trim();
-                    else
-                        release.Title = titleParts[0].Trim();
-
                     var qDetailsLink = qTitle.QuerySelector("a[href^=\"details.php\"]");
                     release.Comments = new Uri(SiteLink + qDetailsLink.GetAttribute("href"));
                     release.Link = new Uri(SiteLink + row.QuerySelector("a[href^=\"download.php\"]").GetAttribute("href"));
                     release.Guid = release.Link;
-
                     var dateString = row.QuerySelector("div:last-child").TextContent.Trim();
-                    var pattern = "\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}";
+                    var match = Regex.Match(dateString, @"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}");
-                    var match = Regex.Match(dateString, pattern);
                     if (match.Success)
-                    {
                         release.PublishDate = DateTime.ParseExact(match.Value, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
-                    }
+                    var sizeStr = row.QuerySelector(".bSize").TextContent.Trim();
-
-                    var sizeStr = row.QuerySelector(".bSize").TextContent;
                     release.Size = ReleaseInfo.GetBytes(sizeStr);
                     release.Seeders = ParseUtil.CoerceInt(row.QuerySelector(".bUping").TextContent.Trim());
                     release.Peers = release.Seeders + ParseUtil.CoerceInt(row.QuerySelector(".bDowning").TextContent.Trim());
-
+                    var files = row.QuerySelector("div.bFiles").ChildNodes.Last().TextContent.Trim();
-                    var files = row.QuerySelector("div.bFiles").LastChild.ToString();
                     release.Files = ParseUtil.CoerceInt(files);
-
+                    var grabs = row.QuerySelector("div.bFinish").ChildNodes.Last().TextContent.Trim();
-                    var grabs = row.QuerySelector("div.bFinish").LastChild.ToString();
                     release.Grabs = ParseUtil.CoerceInt(grabs);
-
+                    release.DownloadVolumeFactor = row.QuerySelector("img[src=\"/pic/free.jpg\"]") != null ? 0 : 1;
-                    if (row.QuerySelector("img[src=\"/pic/free.jpg\"]") != null)
-                        release.DownloadVolumeFactor = 0;
-                    else
-                        release.DownloadVolumeFactor = 1;
-
                     if (row.QuerySelector("img[src=\"/pic/triple.jpg\"]") != null)
                         release.UploadVolumeFactor = 3;
                     else if (row.QuerySelector("img[src=\"/pic/double.jpg\"]") != null)
                         release.UploadVolumeFactor = 2;
                     else
                         release.UploadVolumeFactor = 1;
-
                     releases.Add(release);
                 }
             }