diff --git a/src/Jackett.Common/Indexers/Definitions/NorBits.cs b/src/Jackett.Common/Indexers/Definitions/NorBits.cs index 90cb417cf..64a4d8c85 100644 --- a/src/Jackett.Common/Indexers/Definitions/NorBits.cs +++ b/src/Jackett.Common/Indexers/Definitions/NorBits.cs @@ -5,8 +5,10 @@ using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; using System.Text; +using System.Text.RegularExpressions; using System.Threading.Tasks; using AngleSharp.Dom; +using AngleSharp.Html.Dom; using AngleSharp.Html.Parser; using Jackett.Common.Helpers; using Jackett.Common.Models; @@ -72,18 +74,39 @@ namespace Jackett.Common.Indexers.Definitions }; caps.Categories.AddCategoryMapping("main_cat[]=1", TorznabCatType.Movies, "Filmer"); + caps.Categories.AddCategoryMapping("main_cat[]=1&sub2_cat[]=49", TorznabCatType.MoviesUHD, "Filmer - UHD-2160p"); + caps.Categories.AddCategoryMapping("main_cat[]=1&sub2_cat[]=19", TorznabCatType.MoviesHD, "Filmer - HD-1080p/i"); + caps.Categories.AddCategoryMapping("main_cat[]=1&sub2_cat[]=20", TorznabCatType.MoviesHD, "Filmer - HD-720p"); + caps.Categories.AddCategoryMapping("main_cat[]=1&sub2_cat[]=22", TorznabCatType.MoviesSD, "Filmer - SD"); caps.Categories.AddCategoryMapping("main_cat[]=2", TorznabCatType.TV, "TV"); + caps.Categories.AddCategoryMapping("main_cat[]=2&sub2_cat[]=49", TorznabCatType.TVUHD, "TV - UHD-2160p"); + caps.Categories.AddCategoryMapping("main_cat[]=2&sub2_cat[]=19", TorznabCatType.TVHD, "TV - HD-1080p/i"); + caps.Categories.AddCategoryMapping("main_cat[]=2&sub2_cat[]=20", TorznabCatType.TVHD, "TV - HD-720p"); + caps.Categories.AddCategoryMapping("main_cat[]=2&sub2_cat[]=22", TorznabCatType.TVSD, "TV - SD"); caps.Categories.AddCategoryMapping("main_cat[]=3", TorznabCatType.PC, "Programmer"); caps.Categories.AddCategoryMapping("main_cat[]=4", TorznabCatType.Console, "Spill"); caps.Categories.AddCategoryMapping("main_cat[]=5", TorznabCatType.Audio, "Musikk"); + caps.Categories.AddCategoryMapping("main_cat[]=5&sub2_cat[]=42", TorznabCatType.AudioMP3, "Musikk - 192"); + caps.Categories.AddCategoryMapping("main_cat[]=5&sub2_cat[]=43", TorznabCatType.AudioMP3, "Musikk - 256"); + caps.Categories.AddCategoryMapping("main_cat[]=5&sub2_cat[]=44", TorznabCatType.AudioMP3, "Musikk - 320"); + caps.Categories.AddCategoryMapping("main_cat[]=5&sub2_cat[]=45", TorznabCatType.AudioMP3, "Musikk - VBR"); + caps.Categories.AddCategoryMapping("main_cat[]=5&sub2_cat[]=46", TorznabCatType.AudioLossless, "Musikk - Lossless"); caps.Categories.AddCategoryMapping("main_cat[]=6", TorznabCatType.Books, "Tidsskrift"); caps.Categories.AddCategoryMapping("main_cat[]=7", TorznabCatType.AudioAudiobook, "Lydbøker"); caps.Categories.AddCategoryMapping("main_cat[]=8", TorznabCatType.AudioVideo, "Musikkvideoer"); + caps.Categories.AddCategoryMapping("main_cat[]=8&sub2_cat[]=19", TorznabCatType.AudioVideo, "Musikkvideoer - HD-1080p/i"); + caps.Categories.AddCategoryMapping("main_cat[]=8&sub2_cat[]=20", TorznabCatType.AudioVideo, "Musikkvideoer - HD-720p"); + caps.Categories.AddCategoryMapping("main_cat[]=8&sub2_cat[]=22", TorznabCatType.AudioVideo, "Musikkvideoer - SD"); caps.Categories.AddCategoryMapping("main_cat[]=40", TorznabCatType.AudioOther, "Podcasts"); return caps; } + /// + /// Configure our FADN Provider + /// + /// Our params in Json + /// Configuration state public override async Task ApplyConfiguration(JToken configJson) { // Retrieve config values set by Jackett's user @@ -97,6 +120,10 @@ namespace Jackett.Common.Indexers.Definitions return IndexerConfigurationStatus.RequiresTesting; } + /// + /// Perform login to racker + /// + /// private async Task DoLoginAsync() { // Build WebRequest for index @@ -108,7 +135,7 @@ namespace Jackett.Common.Indexers.Definitions }; // Get index page for cookies - logger.Debug("\nNorBits - Getting index page (for cookies).. with " + SiteLink); + logger.Info("\nNorBits - Getting index page (for cookies).. with " + SiteLink); var indexPage = await webclient.GetResultAsync(myIndexRequest); // Building login form data @@ -136,7 +163,7 @@ namespace Jackett.Common.Indexers.Definitions }; // Get login page -- (not used, but simulation needed by tracker security's checks) - logger.Debug("\nNorBits - Getting login page (user simulation).. with " + LoginUrl); + logger.Info("\nNorBits - Getting login page (user simulation).. with " + LoginUrl); await webclient.GetResultAsync(myRequestLogin); // Build WebRequest for submitting authentication @@ -150,7 +177,7 @@ namespace Jackett.Common.Indexers.Definitions Encoding = Encoding }; - logger.Debug("\nPerform login with " + LoginCheckUrl); + logger.Info("\nPerform login with " + LoginCheckUrl); var response = await webclient.GetResultAsync(request); // Test if we are logged in @@ -162,35 +189,44 @@ namespace Jackett.Common.Indexers.Definitions var redirectTo = response.RedirectingTo; // Oops, unable to login - logger.Debug("NorBits - Login failed: " + message, "error"); + logger.Info("NorBits - Login failed: " + message, "error"); throw new ExceptionWithConfigData("Login failed: " + message, configData); }); - logger.Debug("\nNorBits - Cookies saved for future uses..."); + logger.Info("\nNorBits - Cookies saved for future uses..."); ConfigData.CookieHeader.Value = indexPage.Cookies + " " + response.Cookies + " ts_username=" + ConfigData.Username.Value; - logger.Debug("\nNorBits - Login Success\n"); + logger.Info("\nNorBits - Login Success\n"); } + /// + /// Check logged-in state for provider + /// + /// private async Task CheckLoginAsync() { // Checking ... - logger.Debug("\nNorBits - Checking logged-in state...."); + logger.Info("\nNorBits - Checking logged-in state...."); var loggedInCheck = await RequestWithCookiesAsync(SearchUrl); if (!loggedInCheck.ContentString.Contains("logout.php")) { // Cookie expired, renew session on provider - logger.Debug("NorBits - Not logged, login now...\n"); + logger.Info("NorBits - Not logged, login now...\n"); await DoLoginAsync(); } else { // Already logged, session active - logger.Debug("NorBits - Already logged, continue...\n"); + logger.Info("NorBits - Already logged, continue...\n"); } } + /// + /// Execute our search query + /// + /// Query + /// Releases protected override async Task> PerformQuery(TorznabQuery query) { var releases = new List(); @@ -221,7 +257,7 @@ namespace Jackett.Common.Indexers.Definitions try { - var firstPageRows = dom.QuerySelectorAll("#torrentTable > tbody > tr").Skip(1).ToCollection(); + var firstPageRows = FindTorrentRows(dom); // If pagination available int nbResults; @@ -236,26 +272,29 @@ namespace Jackett.Common.Indexers.Definitions else { // No result found for this query - logger.Debug("\nNorBits - No result found for your query, please try another search term ...\n", "info"); + logger.Info("\nNorBits - No result found for your query, please try another search term ...\n", "info"); break; } - logger.Debug("\nNorBits - Found " + nbResults + " result(s) (+/- " + firstPageRows.Length + ") in " + pageLinkCount + " page(s) for this query !"); - logger.Debug("\nNorBits - There are " + firstPageRows.Length + " results on the first page !"); + logger.Info("\nNorBits - Found " + nbResults + " result(s) (+/- " + firstPageRows.Length + ") in " + pageLinkCount + " page(s) for this query !"); + logger.Info("\nNorBits - There are " + firstPageRows.Length + " results on the first page !"); foreach (var row in firstPageRows) { - var link = new Uri(SiteLink + row.QuerySelector("td:nth-of-type(2) > a[href*=\"download.php?id=\"]")?.GetAttribute("href")?.TrimStart('/')); + var link = new Uri(SiteLink + row.QuerySelector("td:nth-of-type(2) > a[href*=\"download.php?id=\"]")?.GetAttribute("href").TrimStart('/')); var qDetails = row.QuerySelector("td:nth-of-type(2) > a[href*=\"details.php?id=\"]"); - var title = qDetails?.GetAttribute("title")?.Trim(); - var details = new Uri(SiteLink + qDetails?.GetAttribute("href")?.TrimStart('/')); + var title = qDetails?.GetAttribute("title").Trim(); + var details = new Uri(SiteLink + qDetails?.GetAttribute("href").TrimStart('/')); - var catQuery = row.QuerySelector("td:nth-of-type(1) a[href*=\"main_cat[]\"]")?.GetAttribute("href")?.Split('?').Last().Split('&'); - var category = catQuery?.FirstOrDefault(x => x.StartsWith("main_cat[]=", StringComparison.OrdinalIgnoreCase)); + var mainCategory = row.QuerySelector("td:nth-of-type(1) a[href*=\"main_cat[]\"]")?.GetAttribute("href")?.Split('?').Last(); + var secondCategory = row.QuerySelector("td:nth-of-type(1) a[href*=\"sub2_cat[]\"]")?.GetAttribute("href")?.Split('?').Last(); - var seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(9)")?.TextContent); - var leechers = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(10)")?.TextContent); + var categoryList = new[] { mainCategory, secondCategory }; + var cat = string.Join("&", categoryList.Where(c => !string.IsNullOrWhiteSpace(c))); + + var seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(9)").TextContent); + var leechers = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(10)").TextContent); var release = new ReleaseInfo { @@ -263,7 +302,7 @@ namespace Jackett.Common.Indexers.Definitions Details = details, Link = link, Title = title, - Category = MapTrackerCatToNewznab(category), + Category = MapTrackerCatToNewznab(cat), Size = ParseUtil.GetBytes(row.QuerySelector("td:nth-of-type(7)").TextContent), Files = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(3) > a")?.TextContent.Trim()), Grabs = ParseUtil.CoerceLong(row.QuerySelector("td:nth-of-type(8)")?.FirstChild?.TextContent.Trim()), @@ -314,7 +353,15 @@ namespace Jackett.Common.Indexers.Definitions return releases; } - private string BuildQuery(string term, TorznabQuery query, string searchUrl) + /// + /// Build query to process + /// + /// Term to search + /// Torznab Query for categories mapping + /// Search url for provider + /// Page number to request + /// URL to query for parsing and processing results + private string BuildQuery(string term, TorznabQuery query, string searchUrl, int page = 0) { var searchterm = term; @@ -356,17 +403,77 @@ namespace Jackett.Common.Indexers.Definitions searchUrl += "&" + string.Join("&", categoriesList); } - logger.Debug("\nBuilded query for \"" + term + "\"... " + searchUrl); + logger.Info("\nBuilded query for \"" + term + "\"... " + searchUrl); return searchUrl; } + /// + /// Switch Method for Querying + /// + /// URL created by Query Builder + /// Results from query + private async Task QueryExecAsync(string request) + { + WebResult results; + results = await QueryTrackerAsync(request); + return results; + } + + /// + /// Get Torrents Page from Tracker by Query Provided + /// + /// URL created by Query Builder + /// Results from query + private async Task QueryTrackerAsync(string request) + { + // Cache mode not enabled or cached file didn't exist for our query + logger.Info("\nNorBits - Querying tracker for results...."); + + // Request our first page + var results = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value, RequestType.GET, SearchUrl, null); + + // Return results from tracker + return results; + } + + /// + /// Find torrent rows in search pages + /// + /// List of rows + private IHtmlCollection FindTorrentRows(IHtmlDocument dom) => + dom.QuerySelectorAll("#torrentTable > tbody > tr").Skip(1).ToCollection(); + + /// + /// Download torrent file from tracker + /// + /// URL string + /// + public override async Task Download(Uri link) + { + // Retrieving ID from link provided + var id = ParseUtil.CoerceInt(Regex.Match(link.AbsoluteUri, @"\d+").Value); + logger.Info("NorBits - Torrent Requested ID: " + id); + + // Building login form data + var pairs = new Dictionary { + { "torrentid", id.ToString() }, + { "_", string.Empty } // ~~ Strange, blank param... + }; + + // Get torrent file now + var response = await base.Download(link); + + // Return content + return response; + } + /// /// Validate Config entered by user on Jackett /// private void ValidateConfig() { - logger.Debug("\nNorBits - Validating Settings ... \n"); + logger.Info("\nNorBits - Validating Settings ... \n"); // Check Username Setting if (string.IsNullOrEmpty(ConfigData.Username.Value)) @@ -375,7 +482,7 @@ namespace Jackett.Common.Indexers.Definitions } else { - logger.Debug("NorBits - Validated Setting -- Username (auth) => " + ConfigData.Username.Value); + logger.Info("NorBits - Validated Setting -- Username (auth) => " + ConfigData.Username.Value); } // Check Password Setting @@ -385,7 +492,24 @@ namespace Jackett.Common.Indexers.Definitions } else { - logger.Debug("NorBits - Validated Setting -- Password (auth) => " + ConfigData.Password.Value); + logger.Info("NorBits - Validated Setting -- Password (auth) => " + ConfigData.Password.Value); + } + + // Check Max Page Setting + if (!string.IsNullOrEmpty(ConfigData.Pages.Value)) + { + try + { + logger.Info("NorBits - Validated Setting -- Max Pages => " + Convert.ToInt32(ConfigData.Pages.Value)); + } + catch (Exception) + { + throw new ExceptionWithConfigData("Please enter a numeric maximum number of pages to crawl !", ConfigData); + } + } + else + { + throw new ExceptionWithConfigData("Please enter a maximum number of pages to crawl !", ConfigData); } } } diff --git a/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataNorbits.cs b/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataNorbits.cs index d1e5485d0..0b8910fbe 100644 --- a/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataNorbits.cs +++ b/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataNorbits.cs @@ -11,6 +11,7 @@ namespace Jackett.Common.Models.IndexerConfig.Bespoke public StringConfigurationItem TwoFactorAuth { get; private set; } public BoolConfigurationItem freeleech { get; private set; } public DisplayInfoConfigurationItem PagesWarning { get; private set; } + public StringConfigurationItem Pages { get; private set; } public BoolConfigurationItem UseFullSearch { get; private set; } public DisplayInfoConfigurationItem AccountActivity { get; private set; } @@ -22,6 +23,7 @@ namespace Jackett.Common.Models.IndexerConfig.Bespoke TwoFactorAuth = new StringConfigurationItem("2FA Code (Optional)"); freeleech = new BoolConfigurationItem("Search freeleech only") { Value = false }; PagesWarning = new DisplayInfoConfigurationItem("Preferences", "Preferences Configuration (Tweak your search settings),

  • Max Pages to Process let you specify how many page (max) Jackett can process when doing a search. Setting a value higher than 4 is dangerous for you account ! (Result of too many requests to tracker...that will be suspect).
"); + Pages = new StringConfigurationItem("Max Pages to Process (Required)") { Value = "4" }; UseFullSearch = new BoolConfigurationItem("Enable search in description.") { Value = false }; AccountActivity = new DisplayInfoConfigurationItem("Account Inactivity", "Parasites are deactivated after 28 days. (7 days below 0.3 in ratio, 7 days with a warning, 14 days as a parasite). Accounts with nothing downloaded or uploaded will be deactivated after 28 days. This only applies to newly registered accounts without torrent activity (e.g. not passing the norbits test)."); }