From 750b47bca2248d8d0256db4dd7d3a11bf5a2eff7 Mon Sep 17 00:00:00 2001 From: Bogdan Date: Wed, 12 Feb 2025 19:27:45 +0200 Subject: [PATCH] norbits: update category parsing (#15858) Plus minor cleanup Co-authored-by: zodihax --- .../Indexers/Definitions/NorBits.cs | 176 +++--------------- .../Bespoke/ConfigurationDataNorbits.cs | 2 - 2 files changed, 26 insertions(+), 152 deletions(-) diff --git a/src/Jackett.Common/Indexers/Definitions/NorBits.cs b/src/Jackett.Common/Indexers/Definitions/NorBits.cs index 64a4d8c85..90cb417cf 100644 --- a/src/Jackett.Common/Indexers/Definitions/NorBits.cs +++ b/src/Jackett.Common/Indexers/Definitions/NorBits.cs @@ -5,10 +5,8 @@ using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; using System.Text; -using System.Text.RegularExpressions; using System.Threading.Tasks; using AngleSharp.Dom; -using AngleSharp.Html.Dom; using AngleSharp.Html.Parser; using Jackett.Common.Helpers; using Jackett.Common.Models; @@ -74,39 +72,18 @@ namespace Jackett.Common.Indexers.Definitions }; caps.Categories.AddCategoryMapping("main_cat[]=1", TorznabCatType.Movies, "Filmer"); - caps.Categories.AddCategoryMapping("main_cat[]=1&sub2_cat[]=49", TorznabCatType.MoviesUHD, "Filmer - UHD-2160p"); - caps.Categories.AddCategoryMapping("main_cat[]=1&sub2_cat[]=19", TorznabCatType.MoviesHD, "Filmer - HD-1080p/i"); - caps.Categories.AddCategoryMapping("main_cat[]=1&sub2_cat[]=20", TorznabCatType.MoviesHD, "Filmer - HD-720p"); - caps.Categories.AddCategoryMapping("main_cat[]=1&sub2_cat[]=22", TorznabCatType.MoviesSD, "Filmer - SD"); caps.Categories.AddCategoryMapping("main_cat[]=2", TorznabCatType.TV, "TV"); - caps.Categories.AddCategoryMapping("main_cat[]=2&sub2_cat[]=49", TorznabCatType.TVUHD, "TV - UHD-2160p"); - caps.Categories.AddCategoryMapping("main_cat[]=2&sub2_cat[]=19", TorznabCatType.TVHD, "TV - HD-1080p/i"); - caps.Categories.AddCategoryMapping("main_cat[]=2&sub2_cat[]=20", TorznabCatType.TVHD, "TV - HD-720p"); - caps.Categories.AddCategoryMapping("main_cat[]=2&sub2_cat[]=22", TorznabCatType.TVSD, "TV - SD"); caps.Categories.AddCategoryMapping("main_cat[]=3", TorznabCatType.PC, "Programmer"); caps.Categories.AddCategoryMapping("main_cat[]=4", TorznabCatType.Console, "Spill"); caps.Categories.AddCategoryMapping("main_cat[]=5", TorznabCatType.Audio, "Musikk"); - caps.Categories.AddCategoryMapping("main_cat[]=5&sub2_cat[]=42", TorznabCatType.AudioMP3, "Musikk - 192"); - caps.Categories.AddCategoryMapping("main_cat[]=5&sub2_cat[]=43", TorznabCatType.AudioMP3, "Musikk - 256"); - caps.Categories.AddCategoryMapping("main_cat[]=5&sub2_cat[]=44", TorznabCatType.AudioMP3, "Musikk - 320"); - caps.Categories.AddCategoryMapping("main_cat[]=5&sub2_cat[]=45", TorznabCatType.AudioMP3, "Musikk - VBR"); - caps.Categories.AddCategoryMapping("main_cat[]=5&sub2_cat[]=46", TorznabCatType.AudioLossless, "Musikk - Lossless"); caps.Categories.AddCategoryMapping("main_cat[]=6", TorznabCatType.Books, "Tidsskrift"); caps.Categories.AddCategoryMapping("main_cat[]=7", TorznabCatType.AudioAudiobook, "Lydbøker"); caps.Categories.AddCategoryMapping("main_cat[]=8", TorznabCatType.AudioVideo, "Musikkvideoer"); - caps.Categories.AddCategoryMapping("main_cat[]=8&sub2_cat[]=19", TorznabCatType.AudioVideo, "Musikkvideoer - HD-1080p/i"); - caps.Categories.AddCategoryMapping("main_cat[]=8&sub2_cat[]=20", TorznabCatType.AudioVideo, "Musikkvideoer - HD-720p"); - caps.Categories.AddCategoryMapping("main_cat[]=8&sub2_cat[]=22", TorznabCatType.AudioVideo, "Musikkvideoer - SD"); caps.Categories.AddCategoryMapping("main_cat[]=40", TorznabCatType.AudioOther, "Podcasts"); return caps; } - /// - /// Configure our FADN Provider - /// - /// Our params in Json - /// Configuration state public override async Task ApplyConfiguration(JToken configJson) { // Retrieve config values set by Jackett's user @@ -120,10 +97,6 @@ namespace Jackett.Common.Indexers.Definitions return IndexerConfigurationStatus.RequiresTesting; } - /// - /// Perform login to racker - /// - /// private async Task DoLoginAsync() { // Build WebRequest for index @@ -135,7 +108,7 @@ namespace Jackett.Common.Indexers.Definitions }; // Get index page for cookies - logger.Info("\nNorBits - Getting index page (for cookies).. with " + SiteLink); + logger.Debug("\nNorBits - Getting index page (for cookies).. with " + SiteLink); var indexPage = await webclient.GetResultAsync(myIndexRequest); // Building login form data @@ -163,7 +136,7 @@ namespace Jackett.Common.Indexers.Definitions }; // Get login page -- (not used, but simulation needed by tracker security's checks) - logger.Info("\nNorBits - Getting login page (user simulation).. with " + LoginUrl); + logger.Debug("\nNorBits - Getting login page (user simulation).. with " + LoginUrl); await webclient.GetResultAsync(myRequestLogin); // Build WebRequest for submitting authentication @@ -177,7 +150,7 @@ namespace Jackett.Common.Indexers.Definitions Encoding = Encoding }; - logger.Info("\nPerform login with " + LoginCheckUrl); + logger.Debug("\nPerform login with " + LoginCheckUrl); var response = await webclient.GetResultAsync(request); // Test if we are logged in @@ -189,44 +162,35 @@ namespace Jackett.Common.Indexers.Definitions var redirectTo = response.RedirectingTo; // Oops, unable to login - logger.Info("NorBits - Login failed: " + message, "error"); + logger.Debug("NorBits - Login failed: " + message, "error"); throw new ExceptionWithConfigData("Login failed: " + message, configData); }); - logger.Info("\nNorBits - Cookies saved for future uses..."); + logger.Debug("\nNorBits - Cookies saved for future uses..."); ConfigData.CookieHeader.Value = indexPage.Cookies + " " + response.Cookies + " ts_username=" + ConfigData.Username.Value; - logger.Info("\nNorBits - Login Success\n"); + logger.Debug("\nNorBits - Login Success\n"); } - /// - /// Check logged-in state for provider - /// - /// private async Task CheckLoginAsync() { // Checking ... - logger.Info("\nNorBits - Checking logged-in state...."); + logger.Debug("\nNorBits - Checking logged-in state...."); var loggedInCheck = await RequestWithCookiesAsync(SearchUrl); if (!loggedInCheck.ContentString.Contains("logout.php")) { // Cookie expired, renew session on provider - logger.Info("NorBits - Not logged, login now...\n"); + logger.Debug("NorBits - Not logged, login now...\n"); await DoLoginAsync(); } else { // Already logged, session active - logger.Info("NorBits - Already logged, continue...\n"); + logger.Debug("NorBits - Already logged, continue...\n"); } } - /// - /// Execute our search query - /// - /// Query - /// Releases protected override async Task> PerformQuery(TorznabQuery query) { var releases = new List(); @@ -257,7 +221,7 @@ namespace Jackett.Common.Indexers.Definitions try { - var firstPageRows = FindTorrentRows(dom); + var firstPageRows = dom.QuerySelectorAll("#torrentTable > tbody > tr").Skip(1).ToCollection(); // If pagination available int nbResults; @@ -272,29 +236,26 @@ namespace Jackett.Common.Indexers.Definitions else { // No result found for this query - logger.Info("\nNorBits - No result found for your query, please try another search term ...\n", "info"); + logger.Debug("\nNorBits - No result found for your query, please try another search term ...\n", "info"); break; } - logger.Info("\nNorBits - Found " + nbResults + " result(s) (+/- " + firstPageRows.Length + ") in " + pageLinkCount + " page(s) for this query !"); - logger.Info("\nNorBits - There are " + firstPageRows.Length + " results on the first page !"); + logger.Debug("\nNorBits - Found " + nbResults + " result(s) (+/- " + firstPageRows.Length + ") in " + pageLinkCount + " page(s) for this query !"); + logger.Debug("\nNorBits - There are " + firstPageRows.Length + " results on the first page !"); foreach (var row in firstPageRows) { - var link = new Uri(SiteLink + row.QuerySelector("td:nth-of-type(2) > a[href*=\"download.php?id=\"]")?.GetAttribute("href").TrimStart('/')); + var link = new Uri(SiteLink + row.QuerySelector("td:nth-of-type(2) > a[href*=\"download.php?id=\"]")?.GetAttribute("href")?.TrimStart('/')); var qDetails = row.QuerySelector("td:nth-of-type(2) > a[href*=\"details.php?id=\"]"); - var title = qDetails?.GetAttribute("title").Trim(); - var details = new Uri(SiteLink + qDetails?.GetAttribute("href").TrimStart('/')); + var title = qDetails?.GetAttribute("title")?.Trim(); + var details = new Uri(SiteLink + qDetails?.GetAttribute("href")?.TrimStart('/')); - var mainCategory = row.QuerySelector("td:nth-of-type(1) a[href*=\"main_cat[]\"]")?.GetAttribute("href")?.Split('?').Last(); - var secondCategory = row.QuerySelector("td:nth-of-type(1) a[href*=\"sub2_cat[]\"]")?.GetAttribute("href")?.Split('?').Last(); + var catQuery = row.QuerySelector("td:nth-of-type(1) a[href*=\"main_cat[]\"]")?.GetAttribute("href")?.Split('?').Last().Split('&'); + var category = catQuery?.FirstOrDefault(x => x.StartsWith("main_cat[]=", StringComparison.OrdinalIgnoreCase)); - var categoryList = new[] { mainCategory, secondCategory }; - var cat = string.Join("&", categoryList.Where(c => !string.IsNullOrWhiteSpace(c))); - - var seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(9)").TextContent); - var leechers = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(10)").TextContent); + var seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(9)")?.TextContent); + var leechers = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(10)")?.TextContent); var release = new ReleaseInfo { @@ -302,7 +263,7 @@ namespace Jackett.Common.Indexers.Definitions Details = details, Link = link, Title = title, - Category = MapTrackerCatToNewznab(cat), + Category = MapTrackerCatToNewznab(category), Size = ParseUtil.GetBytes(row.QuerySelector("td:nth-of-type(7)").TextContent), Files = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(3) > a")?.TextContent.Trim()), Grabs = ParseUtil.CoerceLong(row.QuerySelector("td:nth-of-type(8)")?.FirstChild?.TextContent.Trim()), @@ -353,15 +314,7 @@ namespace Jackett.Common.Indexers.Definitions return releases; } - /// - /// Build query to process - /// - /// Term to search - /// Torznab Query for categories mapping - /// Search url for provider - /// Page number to request - /// URL to query for parsing and processing results - private string BuildQuery(string term, TorznabQuery query, string searchUrl, int page = 0) + private string BuildQuery(string term, TorznabQuery query, string searchUrl) { var searchterm = term; @@ -403,77 +356,17 @@ namespace Jackett.Common.Indexers.Definitions searchUrl += "&" + string.Join("&", categoriesList); } - logger.Info("\nBuilded query for \"" + term + "\"... " + searchUrl); + logger.Debug("\nBuilded query for \"" + term + "\"... " + searchUrl); return searchUrl; } - /// - /// Switch Method for Querying - /// - /// URL created by Query Builder - /// Results from query - private async Task QueryExecAsync(string request) - { - WebResult results; - results = await QueryTrackerAsync(request); - return results; - } - - /// - /// Get Torrents Page from Tracker by Query Provided - /// - /// URL created by Query Builder - /// Results from query - private async Task QueryTrackerAsync(string request) - { - // Cache mode not enabled or cached file didn't exist for our query - logger.Info("\nNorBits - Querying tracker for results...."); - - // Request our first page - var results = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value, RequestType.GET, SearchUrl, null); - - // Return results from tracker - return results; - } - - /// - /// Find torrent rows in search pages - /// - /// List of rows - private IHtmlCollection FindTorrentRows(IHtmlDocument dom) => - dom.QuerySelectorAll("#torrentTable > tbody > tr").Skip(1).ToCollection(); - - /// - /// Download torrent file from tracker - /// - /// URL string - /// - public override async Task Download(Uri link) - { - // Retrieving ID from link provided - var id = ParseUtil.CoerceInt(Regex.Match(link.AbsoluteUri, @"\d+").Value); - logger.Info("NorBits - Torrent Requested ID: " + id); - - // Building login form data - var pairs = new Dictionary { - { "torrentid", id.ToString() }, - { "_", string.Empty } // ~~ Strange, blank param... - }; - - // Get torrent file now - var response = await base.Download(link); - - // Return content - return response; - } - /// /// Validate Config entered by user on Jackett /// private void ValidateConfig() { - logger.Info("\nNorBits - Validating Settings ... \n"); + logger.Debug("\nNorBits - Validating Settings ... \n"); // Check Username Setting if (string.IsNullOrEmpty(ConfigData.Username.Value)) @@ -482,7 +375,7 @@ namespace Jackett.Common.Indexers.Definitions } else { - logger.Info("NorBits - Validated Setting -- Username (auth) => " + ConfigData.Username.Value); + logger.Debug("NorBits - Validated Setting -- Username (auth) => " + ConfigData.Username.Value); } // Check Password Setting @@ -492,24 +385,7 @@ namespace Jackett.Common.Indexers.Definitions } else { - logger.Info("NorBits - Validated Setting -- Password (auth) => " + ConfigData.Password.Value); - } - - // Check Max Page Setting - if (!string.IsNullOrEmpty(ConfigData.Pages.Value)) - { - try - { - logger.Info("NorBits - Validated Setting -- Max Pages => " + Convert.ToInt32(ConfigData.Pages.Value)); - } - catch (Exception) - { - throw new ExceptionWithConfigData("Please enter a numeric maximum number of pages to crawl !", ConfigData); - } - } - else - { - throw new ExceptionWithConfigData("Please enter a maximum number of pages to crawl !", ConfigData); + logger.Debug("NorBits - Validated Setting -- Password (auth) => " + ConfigData.Password.Value); } } } diff --git a/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataNorbits.cs b/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataNorbits.cs index 0b8910fbe..d1e5485d0 100644 --- a/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataNorbits.cs +++ b/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataNorbits.cs @@ -11,7 +11,6 @@ namespace Jackett.Common.Models.IndexerConfig.Bespoke public StringConfigurationItem TwoFactorAuth { get; private set; } public BoolConfigurationItem freeleech { get; private set; } public DisplayInfoConfigurationItem PagesWarning { get; private set; } - public StringConfigurationItem Pages { get; private set; } public BoolConfigurationItem UseFullSearch { get; private set; } public DisplayInfoConfigurationItem AccountActivity { get; private set; } @@ -23,7 +22,6 @@ namespace Jackett.Common.Models.IndexerConfig.Bespoke TwoFactorAuth = new StringConfigurationItem("2FA Code (Optional)"); freeleech = new BoolConfigurationItem("Search freeleech only") { Value = false }; PagesWarning = new DisplayInfoConfigurationItem("Preferences", "Preferences Configuration (Tweak your search settings),

  • Max Pages to Process let you specify how many page (max) Jackett can process when doing a search. Setting a value higher than 4 is dangerous for you account ! (Result of too many requests to tracker...that will be suspect).
"); - Pages = new StringConfigurationItem("Max Pages to Process (Required)") { Value = "4" }; UseFullSearch = new BoolConfigurationItem("Enable search in description.") { Value = false }; AccountActivity = new DisplayInfoConfigurationItem("Account Inactivity", "Parasites are deactivated after 28 days. (7 days below 0.3 in ratio, 7 days with a warning, 14 days as a parasite). Accounts with nothing downloaded or uploaded will be deactivated after 28 days. This only applies to newly registered accounts without torrent activity (e.g. not passing the norbits test)."); }