mirror of
https://github.com/Jackett/Jackett.git
synced 2025-09-17 17:34:09 +02:00
Revert "norbits: update category parsing (#15858)"
This reverts commit 750b47bca2.
This commit is contained in:
Bogdan
@@ -5,8 +5,10 @@ using System.Diagnostics.CodeAnalysis;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
using AngleSharp.Dom;
|
||||
using AngleSharp.Html.Dom;
|
||||
using AngleSharp.Html.Parser;
|
||||
using Jackett.Common.Helpers;
|
||||
using Jackett.Common.Models;
|
||||
@@ -72,18 +74,39 @@ namespace Jackett.Common.Indexers.Definitions
|
||||
};
|
||||
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=1", TorznabCatType.Movies, "Filmer");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=1&sub2_cat[]=49", TorznabCatType.MoviesUHD, "Filmer - UHD-2160p");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=1&sub2_cat[]=19", TorznabCatType.MoviesHD, "Filmer - HD-1080p/i");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=1&sub2_cat[]=20", TorznabCatType.MoviesHD, "Filmer - HD-720p");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=1&sub2_cat[]=22", TorznabCatType.MoviesSD, "Filmer - SD");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=2", TorznabCatType.TV, "TV");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=2&sub2_cat[]=49", TorznabCatType.TVUHD, "TV - UHD-2160p");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=2&sub2_cat[]=19", TorznabCatType.TVHD, "TV - HD-1080p/i");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=2&sub2_cat[]=20", TorznabCatType.TVHD, "TV - HD-720p");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=2&sub2_cat[]=22", TorznabCatType.TVSD, "TV - SD");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=3", TorznabCatType.PC, "Programmer");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=4", TorznabCatType.Console, "Spill");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=5", TorznabCatType.Audio, "Musikk");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=5&sub2_cat[]=42", TorznabCatType.AudioMP3, "Musikk - 192");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=5&sub2_cat[]=43", TorznabCatType.AudioMP3, "Musikk - 256");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=5&sub2_cat[]=44", TorznabCatType.AudioMP3, "Musikk - 320");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=5&sub2_cat[]=45", TorznabCatType.AudioMP3, "Musikk - VBR");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=5&sub2_cat[]=46", TorznabCatType.AudioLossless, "Musikk - Lossless");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=6", TorznabCatType.Books, "Tidsskrift");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=7", TorznabCatType.AudioAudiobook, "Lydbøker");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=8", TorznabCatType.AudioVideo, "Musikkvideoer");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=8&sub2_cat[]=19", TorznabCatType.AudioVideo, "Musikkvideoer - HD-1080p/i");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=8&sub2_cat[]=20", TorznabCatType.AudioVideo, "Musikkvideoer - HD-720p");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=8&sub2_cat[]=22", TorznabCatType.AudioVideo, "Musikkvideoer - SD");
|
||||
caps.Categories.AddCategoryMapping("main_cat[]=40", TorznabCatType.AudioOther, "Podcasts");
|
||||
|
||||
return caps;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configure our FADN Provider
|
||||
/// </summary>
|
||||
/// <param name="configJson">Our params in Json</param>
|
||||
/// <returns>Configuration state</returns>
|
||||
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
|
||||
{
|
||||
// Retrieve config values set by Jackett's user
|
||||
@@ -97,6 +120,10 @@ namespace Jackett.Common.Indexers.Definitions
|
||||
return IndexerConfigurationStatus.RequiresTesting;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Perform login to racker
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
private async Task DoLoginAsync()
|
||||
{
|
||||
// Build WebRequest for index
|
||||
@@ -108,7 +135,7 @@ namespace Jackett.Common.Indexers.Definitions
|
||||
};
|
||||
|
||||
// Get index page for cookies
|
||||
logger.Debug("\nNorBits - Getting index page (for cookies).. with " + SiteLink);
|
||||
logger.Info("\nNorBits - Getting index page (for cookies).. with " + SiteLink);
|
||||
var indexPage = await webclient.GetResultAsync(myIndexRequest);
|
||||
|
||||
// Building login form data
|
||||
@@ -136,7 +163,7 @@ namespace Jackett.Common.Indexers.Definitions
|
||||
};
|
||||
|
||||
// Get login page -- (not used, but simulation needed by tracker security's checks)
|
||||
logger.Debug("\nNorBits - Getting login page (user simulation).. with " + LoginUrl);
|
||||
logger.Info("\nNorBits - Getting login page (user simulation).. with " + LoginUrl);
|
||||
await webclient.GetResultAsync(myRequestLogin);
|
||||
|
||||
// Build WebRequest for submitting authentication
|
||||
@@ -150,7 +177,7 @@ namespace Jackett.Common.Indexers.Definitions
|
||||
Encoding = Encoding
|
||||
};
|
||||
|
||||
logger.Debug("\nPerform login with " + LoginCheckUrl);
|
||||
logger.Info("\nPerform login with " + LoginCheckUrl);
|
||||
var response = await webclient.GetResultAsync(request);
|
||||
|
||||
// Test if we are logged in
|
||||
@@ -162,35 +189,44 @@ namespace Jackett.Common.Indexers.Definitions
|
||||
var redirectTo = response.RedirectingTo;
|
||||
|
||||
// Oops, unable to login
|
||||
logger.Debug("NorBits - Login failed: " + message, "error");
|
||||
logger.Info("NorBits - Login failed: " + message, "error");
|
||||
throw new ExceptionWithConfigData("Login failed: " + message, configData);
|
||||
});
|
||||
|
||||
logger.Debug("\nNorBits - Cookies saved for future uses...");
|
||||
logger.Info("\nNorBits - Cookies saved for future uses...");
|
||||
ConfigData.CookieHeader.Value = indexPage.Cookies + " " + response.Cookies + " ts_username=" + ConfigData.Username.Value;
|
||||
|
||||
logger.Debug("\nNorBits - Login Success\n");
|
||||
logger.Info("\nNorBits - Login Success\n");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check logged-in state for provider
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
private async Task CheckLoginAsync()
|
||||
{
|
||||
// Checking ...
|
||||
logger.Debug("\nNorBits - Checking logged-in state....");
|
||||
logger.Info("\nNorBits - Checking logged-in state....");
|
||||
var loggedInCheck = await RequestWithCookiesAsync(SearchUrl);
|
||||
if (!loggedInCheck.ContentString.Contains("logout.php"))
|
||||
{
|
||||
// Cookie expired, renew session on provider
|
||||
logger.Debug("NorBits - Not logged, login now...\n");
|
||||
logger.Info("NorBits - Not logged, login now...\n");
|
||||
|
||||
await DoLoginAsync();
|
||||
}
|
||||
else
|
||||
{
|
||||
// Already logged, session active
|
||||
logger.Debug("NorBits - Already logged, continue...\n");
|
||||
logger.Info("NorBits - Already logged, continue...\n");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Execute our search query
|
||||
/// </summary>
|
||||
/// <param name="query">Query</param>
|
||||
/// <returns>Releases</returns>
|
||||
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
|
||||
{
|
||||
var releases = new List<ReleaseInfo>();
|
||||
@@ -221,7 +257,7 @@ namespace Jackett.Common.Indexers.Definitions
|
||||
|
||||
try
|
||||
{
|
||||
var firstPageRows = dom.QuerySelectorAll("#torrentTable > tbody > tr").Skip(1).ToCollection();
|
||||
var firstPageRows = FindTorrentRows(dom);
|
||||
|
||||
// If pagination available
|
||||
int nbResults;
|
||||
@@ -236,26 +272,29 @@ namespace Jackett.Common.Indexers.Definitions
|
||||
else
|
||||
{
|
||||
// No result found for this query
|
||||
logger.Debug("\nNorBits - No result found for your query, please try another search term ...\n", "info");
|
||||
logger.Info("\nNorBits - No result found for your query, please try another search term ...\n", "info");
|
||||
break;
|
||||
}
|
||||
|
||||
logger.Debug("\nNorBits - Found " + nbResults + " result(s) (+/- " + firstPageRows.Length + ") in " + pageLinkCount + " page(s) for this query !");
|
||||
logger.Debug("\nNorBits - There are " + firstPageRows.Length + " results on the first page !");
|
||||
logger.Info("\nNorBits - Found " + nbResults + " result(s) (+/- " + firstPageRows.Length + ") in " + pageLinkCount + " page(s) for this query !");
|
||||
logger.Info("\nNorBits - There are " + firstPageRows.Length + " results on the first page !");
|
||||
|
||||
foreach (var row in firstPageRows)
|
||||
{
|
||||
var link = new Uri(SiteLink + row.QuerySelector("td:nth-of-type(2) > a[href*=\"download.php?id=\"]")?.GetAttribute("href")?.TrimStart('/'));
|
||||
var link = new Uri(SiteLink + row.QuerySelector("td:nth-of-type(2) > a[href*=\"download.php?id=\"]")?.GetAttribute("href").TrimStart('/'));
|
||||
var qDetails = row.QuerySelector("td:nth-of-type(2) > a[href*=\"details.php?id=\"]");
|
||||
|
||||
var title = qDetails?.GetAttribute("title")?.Trim();
|
||||
var details = new Uri(SiteLink + qDetails?.GetAttribute("href")?.TrimStart('/'));
|
||||
var title = qDetails?.GetAttribute("title").Trim();
|
||||
var details = new Uri(SiteLink + qDetails?.GetAttribute("href").TrimStart('/'));
|
||||
|
||||
var catQuery = row.QuerySelector("td:nth-of-type(1) a[href*=\"main_cat[]\"]")?.GetAttribute("href")?.Split('?').Last().Split('&');
|
||||
var category = catQuery?.FirstOrDefault(x => x.StartsWith("main_cat[]=", StringComparison.OrdinalIgnoreCase));
|
||||
var mainCategory = row.QuerySelector("td:nth-of-type(1) a[href*=\"main_cat[]\"]")?.GetAttribute("href")?.Split('?').Last();
|
||||
var secondCategory = row.QuerySelector("td:nth-of-type(1) a[href*=\"sub2_cat[]\"]")?.GetAttribute("href")?.Split('?').Last();
|
||||
|
||||
var seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(9)")?.TextContent);
|
||||
var leechers = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(10)")?.TextContent);
|
||||
var categoryList = new[] { mainCategory, secondCategory };
|
||||
var cat = string.Join("&", categoryList.Where(c => !string.IsNullOrWhiteSpace(c)));
|
||||
|
||||
var seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(9)").TextContent);
|
||||
var leechers = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(10)").TextContent);
|
||||
|
||||
var release = new ReleaseInfo
|
||||
{
|
||||
@@ -263,7 +302,7 @@ namespace Jackett.Common.Indexers.Definitions
|
||||
Details = details,
|
||||
Link = link,
|
||||
Title = title,
|
||||
Category = MapTrackerCatToNewznab(category),
|
||||
Category = MapTrackerCatToNewznab(cat),
|
||||
Size = ParseUtil.GetBytes(row.QuerySelector("td:nth-of-type(7)").TextContent),
|
||||
Files = ParseUtil.CoerceInt(row.QuerySelector("td:nth-of-type(3) > a")?.TextContent.Trim()),
|
||||
Grabs = ParseUtil.CoerceLong(row.QuerySelector("td:nth-of-type(8)")?.FirstChild?.TextContent.Trim()),
|
||||
@@ -314,7 +353,15 @@ namespace Jackett.Common.Indexers.Definitions
|
||||
return releases;
|
||||
}
|
||||
|
||||
private string BuildQuery(string term, TorznabQuery query, string searchUrl)
|
||||
/// <summary>
|
||||
/// Build query to process
|
||||
/// </summary>
|
||||
/// <param name="term">Term to search</param>
|
||||
/// <param name="query">Torznab Query for categories mapping</param>
|
||||
/// <param name="searchUrl">Search url for provider</param>
|
||||
/// <param name="page">Page number to request</param>
|
||||
/// <returns>URL to query for parsing and processing results</returns>
|
||||
private string BuildQuery(string term, TorznabQuery query, string searchUrl, int page = 0)
|
||||
{
|
||||
var searchterm = term;
|
||||
|
||||
@@ -356,17 +403,77 @@ namespace Jackett.Common.Indexers.Definitions
|
||||
searchUrl += "&" + string.Join("&", categoriesList);
|
||||
}
|
||||
|
||||
logger.Debug("\nBuilded query for \"" + term + "\"... " + searchUrl);
|
||||
logger.Info("\nBuilded query for \"" + term + "\"... " + searchUrl);
|
||||
|
||||
return searchUrl;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Switch Method for Querying
|
||||
/// </summary>
|
||||
/// <param name="request">URL created by Query Builder</param>
|
||||
/// <returns>Results from query</returns>
|
||||
private async Task<WebResult> QueryExecAsync(string request)
|
||||
{
|
||||
WebResult results;
|
||||
results = await QueryTrackerAsync(request);
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get Torrents Page from Tracker by Query Provided
|
||||
/// </summary>
|
||||
/// <param name="request">URL created by Query Builder</param>
|
||||
/// <returns>Results from query</returns>
|
||||
private async Task<WebResult> QueryTrackerAsync(string request)
|
||||
{
|
||||
// Cache mode not enabled or cached file didn't exist for our query
|
||||
logger.Info("\nNorBits - Querying tracker for results....");
|
||||
|
||||
// Request our first page
|
||||
var results = await RequestWithCookiesAndRetryAsync(request, ConfigData.CookieHeader.Value, RequestType.GET, SearchUrl, null);
|
||||
|
||||
// Return results from tracker
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Find torrent rows in search pages
|
||||
/// </summary>
|
||||
/// <returns>List of rows</returns>
|
||||
private IHtmlCollection<IElement> FindTorrentRows(IHtmlDocument dom) =>
|
||||
dom.QuerySelectorAll("#torrentTable > tbody > tr").Skip(1).ToCollection();
|
||||
|
||||
/// <summary>
|
||||
/// Download torrent file from tracker
|
||||
/// </summary>
|
||||
/// <param name="link">URL string</param>
|
||||
/// <returns></returns>
|
||||
public override async Task<byte[]> Download(Uri link)
|
||||
{
|
||||
// Retrieving ID from link provided
|
||||
var id = ParseUtil.CoerceInt(Regex.Match(link.AbsoluteUri, @"\d+").Value);
|
||||
logger.Info("NorBits - Torrent Requested ID: " + id);
|
||||
|
||||
// Building login form data
|
||||
var pairs = new Dictionary<string, string> {
|
||||
{ "torrentid", id.ToString() },
|
||||
{ "_", string.Empty } // ~~ Strange, blank param...
|
||||
};
|
||||
|
||||
// Get torrent file now
|
||||
var response = await base.Download(link);
|
||||
|
||||
// Return content
|
||||
return response;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validate Config entered by user on Jackett
|
||||
/// </summary>
|
||||
private void ValidateConfig()
|
||||
{
|
||||
logger.Debug("\nNorBits - Validating Settings ... \n");
|
||||
logger.Info("\nNorBits - Validating Settings ... \n");
|
||||
|
||||
// Check Username Setting
|
||||
if (string.IsNullOrEmpty(ConfigData.Username.Value))
|
||||
@@ -375,7 +482,7 @@ namespace Jackett.Common.Indexers.Definitions
|
||||
}
|
||||
else
|
||||
{
|
||||
logger.Debug("NorBits - Validated Setting -- Username (auth) => " + ConfigData.Username.Value);
|
||||
logger.Info("NorBits - Validated Setting -- Username (auth) => " + ConfigData.Username.Value);
|
||||
}
|
||||
|
||||
// Check Password Setting
|
||||
@@ -385,7 +492,24 @@ namespace Jackett.Common.Indexers.Definitions
|
||||
}
|
||||
else
|
||||
{
|
||||
logger.Debug("NorBits - Validated Setting -- Password (auth) => " + ConfigData.Password.Value);
|
||||
logger.Info("NorBits - Validated Setting -- Password (auth) => " + ConfigData.Password.Value);
|
||||
}
|
||||
|
||||
// Check Max Page Setting
|
||||
if (!string.IsNullOrEmpty(ConfigData.Pages.Value))
|
||||
{
|
||||
try
|
||||
{
|
||||
logger.Info("NorBits - Validated Setting -- Max Pages => " + Convert.ToInt32(ConfigData.Pages.Value));
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
throw new ExceptionWithConfigData("Please enter a numeric maximum number of pages to crawl !", ConfigData);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new ExceptionWithConfigData("Please enter a maximum number of pages to crawl !", ConfigData);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ namespace Jackett.Common.Models.IndexerConfig.Bespoke
|
||||
public StringConfigurationItem TwoFactorAuth { get; private set; }
|
||||
public BoolConfigurationItem freeleech { get; private set; }
|
||||
public DisplayInfoConfigurationItem PagesWarning { get; private set; }
|
||||
public StringConfigurationItem Pages { get; private set; }
|
||||
public BoolConfigurationItem UseFullSearch { get; private set; }
|
||||
public DisplayInfoConfigurationItem AccountActivity { get; private set; }
|
||||
|
||||
@@ -22,6 +23,7 @@ namespace Jackett.Common.Models.IndexerConfig.Bespoke
|
||||
TwoFactorAuth = new StringConfigurationItem("2FA Code (Optional)");
|
||||
freeleech = new BoolConfigurationItem("Search freeleech only") { Value = false };
|
||||
PagesWarning = new DisplayInfoConfigurationItem("Preferences", "<b>Preferences Configuration</b> (<i>Tweak your search settings</i>),<br /><br /> <ul><li><b>Max Pages to Process</b> let you specify how many page (max) Jackett can process when doing a search. Setting a value <b>higher than 4 is dangerous</b> for you account ! (<b>Result of too many requests to tracker...that <u>will be suspect</u></b>).</li></ul>");
|
||||
Pages = new StringConfigurationItem("Max Pages to Process (Required)") { Value = "4" };
|
||||
UseFullSearch = new BoolConfigurationItem("Enable search in description.") { Value = false };
|
||||
AccountActivity = new DisplayInfoConfigurationItem("Account Inactivity", "Parasites are deactivated after 28 days. (7 days below 0.3 in ratio, 7 days with a warning, 14 days as a parasite). Accounts with nothing downloaded or uploaded will be deactivated after 28 days. This only applies to newly registered accounts without torrent activity (e.g. not passing the norbits test).");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user