assorted: use GetArgumentFromQueryString and other minor fixes (#13941)

src/Jackett.Common/Indexers/HDSpace.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Collections.Specialized;
 using System.Diagnostics.CodeAnalysis;
 using System.Linq;
+using System.Net;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
@@ -11,10 +12,10 @@ using Jackett.Common.Models;
 using Jackett.Common.Models.IndexerConfig;
 using Jackett.Common.Services.Interfaces;
 using Jackett.Common.Utils;
-using Jackett.Common.Utils.Clients;
 using Newtonsoft.Json.Linq;
 using NLog;
 using static Jackett.Common.Models.IndexerConfig.ConfigurationData;
+using WebClient = Jackett.Common.Utils.Clients.WebClient;
 
 namespace Jackett.Common.Indexers
 {
@@ -22,7 +23,7 @@ namespace Jackett.Common.Indexers
     public class HDSpace : BaseWebIndexer
     {
         private string LoginUrl => SiteLink + "index.php?page=login";
-        private string SearchUrl => SiteLink + "index.php?page=torrents&";
+        private string SearchUrl => SiteLink + "index.php?page=torrents";
 
         private new ConfigurationDataBasicLogin configData => (ConfigurationDataBasicLogin)base.configData;
 
@@ -121,17 +122,16 @@ namespace Jackett.Common.Indexers
 
             if (query.IsImdbQuery)
             {
-                queryCollection.Add("options", "2");
+                queryCollection.Set("options", "2");
-                queryCollection.Add("search", query.ImdbIDShort);
+                queryCollection.Set("search", query.ImdbIDShort);
             }
             else
             {
-                queryCollection.Add("options", "0");
+                queryCollection.Set("options", "0");
-                queryCollection.Add("search", query.GetQueryString());
+                queryCollection.Set("search", query.GetQueryString().Replace(".", " "));
             }
 
-            // remove . as not used in titles
+            var response = await RequestWithCookiesAndRetryAsync($"{SearchUrl}&{queryCollection.GetQueryString()}");
-            var response = await RequestWithCookiesAndRetryAsync(SearchUrl + queryCollection.GetQueryString().Replace(".", " "));
 
             try
             {
@@ -156,6 +156,11 @@ namespace Jackett.Common.Indexers
                     release.Title = qLink.TextContent.Trim();
                     release.Details = new Uri(SiteLink + qLink.GetAttribute("href"));
                     release.Guid = release.Details;
+                    release.Link = new Uri(SiteLink + row.Children[3].FirstElementChild.GetAttribute("href"));
+
+                    var torrentTitle = ParseUtil.GetArgumentFromQueryString(release.Link.ToString(), "f")?.Replace(".torrent", "").Trim();
+                    if (!string.IsNullOrWhiteSpace(torrentTitle))
+                        release.Title = WebUtility.HtmlDecode(torrentTitle);
 
                     var qGenres = row.QuerySelector("span[style=\"color: #000000 \"]");
                     var description = "";
@@ -166,9 +171,6 @@ namespace Jackett.Common.Indexers
                     if (imdbLink != null)
                         release.Imdb = ParseUtil.GetImdbID(imdbLink.GetAttribute("href").Split('/').Last());
 
-                    var qDownload = row.Children[3].FirstElementChild;
-                    release.Link = new Uri(SiteLink + qDownload.GetAttribute("href"));
-
                     var dateStr = row.Children[4].TextContent.Trim();
                     //"July 11, 2015, 13:34:09", "Today|Yesterday at 20:04:23"
                     release.PublishDate = DateTimeUtil.FromUnknown(dateStr);
@@ -188,8 +190,9 @@ namespace Jackett.Common.Indexers
                     else
                         release.DownloadVolumeFactor = 1;
                     release.UploadVolumeFactor = 1;
-                    var qCat = row.QuerySelector("a[href^=\"index.php?page=torrents&category=\"]");
+
-                    var cat = qCat.GetAttribute("href").Split('=')[2];
+                    var categoryLink = row.QuerySelector("a[href^=\"index.php?page=torrents&category=\"]").GetAttribute("href");
+                    var cat = ParseUtil.GetArgumentFromQueryString(categoryLink, "category");
                     release.Category = MapTrackerCatToNewznab(cat);
 
                     release.Description = description;

src/Jackett.Common/Indexers/HDTorrents.cs

@@ -170,15 +170,14 @@ namespace Jackett.Common.Indexers
                     var poster = posterMatch.Success ? new Uri(SiteLink + posterMatch.Groups[1].Value.Replace("\\", "/")) : null;
 
                     var link = new Uri(SiteLink + row.Children[4].FirstElementChild.GetAttribute("href"));
-                    var description = row.Children[2].QuerySelector("span").TextContent;
+                    var description = row.Children[2].QuerySelector("span")?.TextContent.Trim();
                     var size = ReleaseInfo.GetBytes(row.Children[7].TextContent);
 
-                    var dateTag = row.Children[6].FirstElementChild;
+                    var dateAdded = string.Join(" ", row.Children[6].FirstElementChild.Attributes.Select(a => a.Name).Take(4));
-                    var dateString = string.Join(" ", dateTag.Attributes.Select(attr => attr.Name));
+                    var publishDate = DateTime.ParseExact(dateAdded, "dd MMM yyyy HH:mm:ss", CultureInfo.InvariantCulture);
-                    var publishDate = DateTime.ParseExact(dateString, "dd MMM yyyy HH:mm:ss zz00", CultureInfo.InvariantCulture).ToLocalTime();
 
-                    var catStr = row.FirstElementChild.FirstElementChild.GetAttribute("href").Split('=')[1];
+                    var categoryLink = row.FirstElementChild.FirstElementChild.GetAttribute("href");
-                    var cat = MapTrackerCatToNewznab(catStr);
+                    var cat = ParseUtil.GetArgumentFromQueryString(categoryLink, "category");
 
                     // Sometimes the uploader column is missing, so seeders, leechers, and grabs may be at a different index.
                     // There's room for improvement, but this works for now.
@@ -230,7 +229,7 @@ namespace Jackett.Common.Indexers
                         Guid = details,
                         Link = link,
                         PublishDate = publishDate,
-                        Category = cat,
+                        Category = MapTrackerCatToNewznab(cat),
                         Description = description,
                         Poster = poster,
                         Imdb = imdb,

src/Jackett.Common/Indexers/ImmortalSeed.cs

@@ -5,6 +5,7 @@ using System.Globalization;
 using System.Linq;
 using System.Net;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using AngleSharp.Html.Parser;
 using Jackett.Common.Models;
@@ -22,6 +23,7 @@ namespace Jackett.Common.Indexers
         private string BrowsePage => SiteLink + "browse.php";
         private string LoginUrl => SiteLink + "takelogin.php";
         private string QueryString => "?do=search&keywords={0}&search_type=t_name&category=0&include_dead_torrents=no";
+        private readonly Regex _dateMatchRegex = new Regex(@"\d{4}-\d{2}-\d{2} \d{2}:\d{2} [AaPp][Mm]", RegexOptions.Compiled);
 
         public override string[] LegacySiteLinks { get; protected set; } = {
             "http://immortalseed.me/"
@@ -189,25 +191,25 @@ namespace Jackett.Common.Indexers
                     release.Details = new Uri(qDetails.GetAttribute("href"));
 
                     // 2021-03-17 03:39 AM
-                    var dateString = row.QuerySelectorAll("td:nth-of-type(2) div").Last().LastChild.TextContent.Trim();
+                    // requests can be 'Pre Release Time: 2013-04-22 02:00 AM Uploaded: 3 Years, 6 Months, 4 Weeks, 2 Days, 16 Hours, 52 Minutes, 41 Seconds after Pre'
-                    release.PublishDate = DateTime.ParseExact(dateString, "yyyy-MM-dd hh:mm tt", CultureInfo.InvariantCulture);
+                    var dateMatch = _dateMatchRegex.Match(row.QuerySelector("td:nth-of-type(2) > div:last-child").TextContent.Trim());
-
+                    if (dateMatch.Success)
-                    var sizeStr = row.QuerySelector("td:nth-of-type(5)").TextContent.Trim();
+                        release.PublishDate = DateTime.ParseExact(dateMatch.Value, "yyyy-MM-dd hh:mm tt", CultureInfo.InvariantCulture);
-                    release.Size = ReleaseInfo.GetBytes(sizeStr);
 
+                    release.Size = ReleaseInfo.GetBytes(row.QuerySelector("td:nth-of-type(5)").TextContent.Trim());
                     release.Seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(7)").TextContent.Trim());
                     release.Peers = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(8)").TextContent.Trim()) + release.Seeders;
 
-                    var catLink = row.QuerySelector("td:nth-of-type(1) a").GetAttribute("href");
+                    var categoryLink = row.QuerySelector("td:nth-of-type(1) a").GetAttribute("href");
-                    var catSplit = catLink.IndexOf("category=");
+                    var cat = ParseUtil.GetArgumentFromQueryString(categoryLink, "category");
-                    if (catSplit > -1)
+                    release.Category = MapTrackerCatToNewznab(cat);
-                        catLink = catLink.Substring(catSplit + 9);
-
-                    release.Category = MapTrackerCatToNewznab(catLink);
 
                     var grabs = row.QuerySelector("td:nth-child(6)").TextContent;
                     release.Grabs = ParseUtil.CoerceInt(grabs);
 
+                    var cover = row.QuerySelector("td:nth-of-type(2) > div > img[src]")?.GetAttribute("src")?.Trim();
+                    release.Poster = !string.IsNullOrEmpty(cover) && cover.StartsWith("/") ? new Uri(SiteLink + cover.TrimStart('/')) : null;
+
                     if (row.QuerySelector("img[title^=\"Free Torrent\"]") != null)
                         release.DownloadVolumeFactor = 0;
                     else if (row.QuerySelector("img[title^=\"Silver Torrent\"]") != null)
@@ -215,10 +217,7 @@ namespace Jackett.Common.Indexers
                     else
                         release.DownloadVolumeFactor = 1;
 
-                    if (row.QuerySelector("img[title^=\"x2 Torrent\"]") != null)
+                    release.UploadVolumeFactor = row.QuerySelector("img[title^=\"x2 Torrent\"]") != null ? 2 : 1;
-                        release.UploadVolumeFactor = 2;
-                    else
-                        release.UploadVolumeFactor = 1;
 
                     releases.Add(release);
                 }

src/Jackett.Common/Indexers/SceneTime.cs

@@ -2,9 +2,9 @@ using System;
 using System.Collections.Generic;
 using System.Collections.Specialized;
 using System.Diagnostics.CodeAnalysis;
+using System.Globalization;
 using System.Linq;
 using System.Text;
-using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using AngleSharp.Html.Parser;
 using Jackett.Common.Models;
@@ -57,7 +57,7 @@ namespace Jackett.Common.Indexers
                    cacheService: cs,
                    configData: new ConfigurationDataSceneTime())
         {
-            Encoding = Encoding.GetEncoding("iso-8859-1");
+            Encoding = Encoding.UTF8;
             Language = "en-US";
             Type = "private";
 
@@ -119,14 +119,14 @@ namespace Jackett.Common.Indexers
 
             var catList = MapTorznabCapsToTrackers(query);
             foreach (var cat in catList)
-                qParams.Add("c" + cat, "1");
+                qParams.Set($"c{cat}", "1");
 
             if (!string.IsNullOrEmpty(query.SanitizedSearchTerm))
-                qParams.Add("search", query.GetQueryString());
+                qParams.Set("search", query.GetQueryString());
 
             // If Only Freeleech Enabled
             if (configData.Freeleech.Value)
-                qParams.Add("freeleech", "on");
+                qParams.Set("freeleech", "on");
 
             var searchUrl = SearchUrl + "?" + qParams.GetQueryString();
             var results = await RequestWithCookiesAsync(searchUrl);
@@ -161,8 +161,7 @@ namespace Jackett.Common.Indexers
                 if (table == null)
                     return releases; // no results
 
-                var headerColumns = table.QuerySelectorAll("tbody > tr > td.cat_Head")
+                var headerColumns = table.QuerySelectorAll("tbody > tr > td.cat_Head").Select(x => x.TextContent).ToList();
-                                         .Select(x => x.TextContent).ToList();
                 var categoryIndex = headerColumns.FindIndex(x => x.Equals("Type"));
                 var nameIndex = headerColumns.FindIndex(x => x.Equals("Name"));
                 var sizeIndex = headerColumns.FindIndex(x => x.Equals("Size"));
@@ -172,44 +171,44 @@ namespace Jackett.Common.Indexers
                 var rows = dom.QuerySelectorAll("tr.browse");
                 foreach (var row in rows)
                 {
-                    // TODO convert to initializer
+                    var qDescCol = row.Children[nameIndex];
+                    var qLink = qDescCol.QuerySelector("a");
+                    var title = qLink.TextContent;
+
+                    if (!query.MatchQueryStringAND(title))
+                        continue;
+
+                    var details = new Uri(SiteLink + "/" + qLink.GetAttribute("href"));
+                    var torrentId = ParseUtil.GetArgumentFromQueryString(qLink.GetAttribute("href"), "id");
+                    var link = new Uri(string.Format(DownloadUrl, torrentId));
+
+                    var categoryLink = row.Children[categoryIndex].QuerySelector("a")?.GetAttribute("href");
+                    var cat = categoryLink != null ? ParseUtil.GetArgumentFromQueryString(categoryLink, "cat") : "82"; // default
+
+                    var seeders = ParseUtil.CoerceInt(row.Children[seedersIndex].TextContent.Trim());
+
+                    var dateAdded = qDescCol.QuerySelector("span[class=\"elapsedDate\"]").GetAttribute("title").Trim();
+                    var publishDate = DateTime.TryParseExact(dateAdded, "dddd, MMMM d, yyyy \\a\\t h:mmtt", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var date)
+                        ? date
+                        : DateTimeUtil.FromTimeAgo(qDescCol.QuerySelector("span[class=\"elapsedDate\"]").TextContent.Trim());
+
                     var release = new ReleaseInfo
                     {
+                        Guid = details,
+                        Details = details,
+                        Link = link,
+                        Title = title,
+                        Category = MapTrackerCatToNewznab(cat),
+                        PublishDate = publishDate,
+                        Size = ReleaseInfo.GetBytes(row.Children[sizeIndex].TextContent),
+                        Seeders = seeders,
+                        Peers = ParseUtil.CoerceInt(row.Children[leechersIndex].TextContent.Trim()) + seeders,
+                        DownloadVolumeFactor = row.QuerySelector("font > b:contains(Freeleech)") != null ? 0 : 1,
+                        UploadVolumeFactor = 1,
                         MinimumRatio = 1,
                         MinimumSeedTime = 259200 // 72 hours
                     };
 
-                    var catId = "82"; // default
-                    var qCatLink = row.Children[categoryIndex].QuerySelector("a");
-                    if (qCatLink != null)
-                    {
-                        catId = new Regex(@"\?cat=(\d*)").Match(qCatLink.GetAttribute("href")).Groups[1].ToString().Trim();
-                    }
-                    release.Category = MapTrackerCatToNewznab(catId);
-
-                    var qDescCol = row.Children[nameIndex];
-                    var qLink = qDescCol.QuerySelector("a");
-                    release.Title = qLink.TextContent;
-                    if (!query.MatchQueryStringAND(release.Title))
-                        continue;
-
-                    release.Details = new Uri(SiteLink + "/" + qLink.GetAttribute("href"));
-                    release.Guid = release.Details;
-
-                    var torrentId = qLink.GetAttribute("href").Split('=')[1];
-                    release.Link = new Uri(string.Format(DownloadUrl, torrentId));
-
-                    release.PublishDate = DateTimeUtil.FromTimeAgo(qDescCol.ChildNodes.Last().TextContent);
-
-                    var sizeStr = row.Children[sizeIndex].TextContent;
-                    release.Size = ReleaseInfo.GetBytes(sizeStr);
-
-                    release.Seeders = ParseUtil.CoerceInt(row.Children[seedersIndex].TextContent.Trim());
-                    release.Peers = ParseUtil.CoerceInt(row.Children[leechersIndex].TextContent.Trim()) + release.Seeders;
-
-                    release.DownloadVolumeFactor = row.QuerySelector("font > b:contains(Freeleech)") != null ? 0 : 1;
-                    release.UploadVolumeFactor = 1;
-
                     releases.Add(release);
                 }
             }

src/Jackett.Common/Indexers/XSpeeds.cs

@@ -25,8 +25,7 @@ namespace Jackett.Common.Indexers
         private string LoginUrl => SiteLink + "takelogin.php";
         private string GetRSSKeyUrl => SiteLink + "getrss.php";
         private string SearchUrl => SiteLink + "browse.php";
-        private readonly Regex _dateMatchRegex = new Regex(
+        private readonly Regex _dateMatchRegex = new Regex(@"\d{2}-\d{2}-\d{4} \d{2}:\d{2}", RegexOptions.Compiled);
-            @"\d{2}-\d{2}-\d{4} \d{2}:\d{2}", RegexOptions.Compiled);
 
         private new ConfigurationDataBasicLoginWithRSSAndDisplay configData =>
             (ConfigurationDataBasicLoginWithRSSAndDisplay)base.configData;
@@ -270,13 +269,13 @@ namespace Jackett.Common.Indexers
                     release.Guid = new Uri(row.QuerySelector("td:nth-of-type(3) a").GetAttribute("href"));
                     release.Link = release.Guid;
                     release.Details = new Uri(qDetails.GetAttribute("href"));
-                    //08-08-2015 12:51
+
+                    // 08-08-2015 12:51
                     // requests can be 'Pre Release Time: 25-04-2021 15:00 Uploaded: 3 Weeks, 2 Days, 23 Hours, 53 Minutes, 39 Seconds after Pre'
-                    var dateMatch = _dateMatchRegex.Match(row.QuerySelectorAll("td:nth-of-type(2) div").Last().TextContent.Trim());
+                    var dateMatch = _dateMatchRegex.Match(row.QuerySelector("td:nth-of-type(2) > div:last-child").TextContent.Trim());
                     if (dateMatch.Success)
-                        release.PublishDate = DateTime.ParseExact(dateMatch.Value
+                        release.PublishDate = DateTime.ParseExact(dateMatch.Value, "dd-MM-yyyy HH:mm", CultureInfo.InvariantCulture);
-                        , "dd-MM-yyyy H:mm",
+
-                        CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal);
                     release.Seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(7)").TextContent);
                     release.Peers = release.Seeders + ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(8)").TextContent.Trim());
                     release.Size = ReleaseInfo.GetBytes(row.QuerySelector("td:nth-of-type(5)").TextContent.Trim());
@@ -285,15 +284,16 @@ namespace Jackett.Common.Indexers
                     if (qPoster != null)
                         release.Poster = new Uri(qPoster.GetAttribute("src"));
 
-                    var cat = row.QuerySelector("td:nth-of-type(1) a").GetAttribute("href");
+                    var categoryLink = row.QuerySelector("td:nth-of-type(1) a").GetAttribute("href");
-                    var catSplit = cat.LastIndexOf('=');
+                    var cat = ParseUtil.GetArgumentFromQueryString(categoryLink, "category");
-                    if (catSplit > -1)
-                        cat = cat.Substring(catSplit + 1);
                     release.Category = MapTrackerCatToNewznab(cat);
 
                     var grabs = row.QuerySelector("td:nth-child(6)").TextContent;
                     release.Grabs = ParseUtil.CoerceInt(grabs);
 
+                    var cover = row.QuerySelector("td:nth-of-type(2) > div > img[src]")?.GetAttribute("src")?.Trim();
+                    release.Poster = !string.IsNullOrEmpty(cover) && cover.StartsWith("http") ? new Uri(cover) : null;
+
                     if (row.QuerySelector("img[alt^=\"Free Torrent\"]") != null)
                         release.DownloadVolumeFactor = 0;
                     else if (row.QuerySelector("img[alt^=\"Silver Torrent\"]") != null)