diff --git a/src/Jackett.Common/Indexers/NCore.cs b/src/Jackett.Common/Indexers/NCore.cs index c197dbb2d..407719706 100644 --- a/src/Jackett.Common/Indexers/NCore.cs +++ b/src/Jackett.Common/Indexers/NCore.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Collections.Specialized; using System.Globalization; using System.Linq; using System.Text; @@ -11,6 +12,7 @@ using Jackett.Common.Models.IndexerConfig.Bespoke; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; using Jackett.Common.Utils.Clients; +using Microsoft.AspNetCore.WebUtilities; using Newtonsoft.Json.Linq; using NLog; @@ -20,29 +22,42 @@ namespace Jackett.Common.Indexers { private string LoginUrl => SiteLink + "login.php"; private string SearchUrl => SiteLink + "torrents.php"; - private readonly string[] LanguageCats = { "xvidser", "dvdser", "hdser", "xvid", "dvd", "dvd9", "hd", "mp3", "lossless", "ebook" }; - private new ConfigurationDataNCore configData + private new ConfigurationDataNCore configData => (ConfigurationDataNCore)base.configData; + + private readonly string[] _languageCats = { - get => (ConfigurationDataNCore)base.configData; - set => base.configData = value; - } + "xvidser", + "dvdser", + "hdser", + "xvid", + "dvd", + "dvd9", + "hd", + "mp3", + "lossless", + "ebook" + }; - public NCore(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps) - : base(name: "nCore", - description: "A Hungarian private torrent site.", - link: "https://ncore.cc/", - caps: new TorznabCapabilities(), - configService: configService, - client: wc, - logger: l, - p: ps, - configData: new ConfigurationDataNCore()) + public NCore(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps) : + base("nCore", + description: "A Hungarian private torrent site.", + link: "https://ncore.cc/", + caps: new TorznabCapabilities + { + SupportsImdbMovieSearch = true, + // supported by the site but disabled due to #8107 + // SupportsImdbTVSearch = true + }, + configService: configService, + client: wc, + logger: l, + p: ps, + configData: new ConfigurationDataNCore()) { Encoding = Encoding.UTF8; Language = "hu-hu"; Type = "private"; - AddCategoryMapping("xvid_hun", TorznabCatType.MoviesSD, "Film SD/HU"); AddCategoryMapping("xvid", TorznabCatType.MoviesSD, "Film SD/EN"); AddCategoryMapping("dvd_hun", TorznabCatType.MoviesDVD, "Film DVDR/HU"); @@ -51,33 +66,27 @@ namespace Jackett.Common.Indexers AddCategoryMapping("dvd9", TorznabCatType.MoviesDVD, "Film DVD9/EN"); AddCategoryMapping("hd_hun", TorznabCatType.MoviesHD, "Film HD/HU"); AddCategoryMapping("hd", TorznabCatType.MoviesHD, "Film HD/EN"); - AddCategoryMapping("xvidser_hun", TorznabCatType.TVSD, "Sorozat SD/HU"); AddCategoryMapping("xvidser", TorznabCatType.TVSD, "Sorozat SD/EN"); AddCategoryMapping("dvdser_hun", TorznabCatType.TVSD, "Sorozat DVDR/HU"); AddCategoryMapping("dvdser", TorznabCatType.TVSD, "Sorozat DVDR/EN"); AddCategoryMapping("hdser_hun", TorznabCatType.TVHD, "Sorozat HD/HU"); AddCategoryMapping("hdser", TorznabCatType.TVHD, "Sorozat HD/EN"); - AddCategoryMapping("mp3_hun", TorznabCatType.AudioMP3, "Zene MP3/HU"); AddCategoryMapping("mp3", TorznabCatType.AudioMP3, "Zene MP3/EN"); AddCategoryMapping("lossless_hun", TorznabCatType.AudioLossless, "Zene Lossless/HU"); AddCategoryMapping("lossless", TorznabCatType.AudioLossless, "Zene Lossless/EN"); AddCategoryMapping("clip", TorznabCatType.AudioVideo, "Zene Klip"); - AddCategoryMapping("xxx_xvid", TorznabCatType.XXXXviD, "XXX SD"); AddCategoryMapping("xxx_dvd", TorznabCatType.XXXDVD, "XXX DVDR"); AddCategoryMapping("xxx_imageset", TorznabCatType.XXXImageset, "XXX Imageset"); AddCategoryMapping("xxx_hd", TorznabCatType.XXX, "XXX HD"); - AddCategoryMapping("game_iso", TorznabCatType.PCGames, "Játék PC/ISO"); AddCategoryMapping("game_rip", TorznabCatType.PCGames, "Játék PC/RIP"); AddCategoryMapping("console", TorznabCatType.Console, "Játék Konzol"); - AddCategoryMapping("iso", TorznabCatType.PCISO, "Program Prog/ISO"); AddCategoryMapping("misc", TorznabCatType.PC0day, "Program Prog/RIP"); AddCategoryMapping("mobil", TorznabCatType.PCPhoneOther, "Program Prog/Mobil"); - AddCategoryMapping("ebook_hun", TorznabCatType.Books, "Könyv eBook/HU"); AddCategoryMapping("ebook", TorznabCatType.Books, "Könyv eBook/EN"); } @@ -85,246 +94,238 @@ namespace Jackett.Common.Indexers public override async Task ApplyConfiguration(JToken configJson) { LoadValuesFromJson(configJson); - if (configData.Hungarian.Value == false && configData.English.Value == false) - throw new ExceptionWithConfigData("Please select atleast one language.", configData); - + throw new ExceptionWithConfigData("Please select at least one language.", configData); var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty); - var pairs = new Dictionary { - { "nev", configData.Username.Value }, - { "pass", configData.Password.Value }, - { "ne_leptessen_ki", "1"}, - { "set_lang", "en" }, - { "submitted", "1" }, - { "submit", "Access!" } + var pairs = new Dictionary + { + {"nev", configData.Username.Value}, + {"pass", configData.Password.Value}, + {"ne_leptessen_ki", "1"}, + {"set_lang", "en"}, + {"submitted", "1"}, + {"submit", "Access!"} }; - if (!string.IsNullOrEmpty(configData.TwoFactor.Value)) pairs.Add("2factor", configData.TwoFactor.Value); - var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, referer: SiteLink); - await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("profile.php"), () => - { - var parser = new HtmlParser(); - var dom = parser.ParseDocument(result.Content); - var messageEl = dom.QuerySelector("#hibauzenet table tbody tr"); - var msgContainer = messageEl.Children[1]; - var errorMessage = msgContainer != null ? msgContainer.TextContent : "Error while trying to login."; - throw new ExceptionWithConfigData(errorMessage, configData); - }); - + await ConfigureIfOK( + result.Cookies, result.Content?.Contains("profile.php") == true, () => + { + var parser = new HtmlParser(); + var dom = parser.ParseDocument(result.Content); + var msgContainer = dom.QuerySelector("#hibauzenet table tbody tr")?.Children[1]; + throw new ExceptionWithConfigData(msgContainer?.TextContent ?? "Error while trying to login.", configData); + }); return IndexerConfigurationStatus.RequiresTesting; } protected override async Task> PerformQuery(TorznabQuery query) { - var results = await PerformQuery(query, null); - if (results.Count() == 0 && query.IsTVSearch) // if we search for a localized title ncore can't handle any extra S/E information, search without it and AND filter the results. See #1450 - { - results = await PerformQuery(query, query.GetEpisodeSearchString()); - } + var results = new List(); + if (!(query.IsImdbQuery && query.IsTVSearch)) + results = await PerformQueryAsync(query, null); + // if we search for a localized title nCore can't handle any extra S/E information + // search without it and AND filter the results. See #1450 + if (query.IsTVSearch && (!results.Any() || query.IsImdbQuery)) + results = await PerformQueryAsync(query, query.GetEpisodeSearchString()); return results; } - private async Task> PerformQuery(TorznabQuery query, string seasonep) + private async Task> PerformQueryAsync(TorznabQuery query, string episodeString) { var releases = new List(); - var searchString = query.GetQueryString(); - var pairs = new List>(); - - if (seasonep != null) - searchString = query.SanitizedSearchTerm; - - pairs.Add(new KeyValuePair("nyit_sorozat_resz", "true")); - pairs.Add(new KeyValuePair("miben", "name")); - pairs.Add(new KeyValuePair("tipus", "kivalasztottak_kozott")); - pairs.Add(new KeyValuePair("submit.x", "1")); - pairs.Add(new KeyValuePair("submit.y", "1")); - pairs.Add(new KeyValuePair("submit", "Ok")); - pairs.Add(new KeyValuePair("mire", searchString)); - - var cats = MapTorznabCapsToTrackers(query); - - if (cats.Count == 0) - cats = GetAllTrackerCategories(); - - foreach (var lcat in LanguageCats) + var pairs = new NameValueCollection { - if (!configData.Hungarian.Value) - cats.Remove(lcat + "_hun"); - if (!configData.English.Value) - cats.Remove(lcat); + {"nyit_sorozat_resz", "true"}, + {"tipus", "kivalasztottak_kozott"}, + {"submit.x", "1"}, + {"submit.y", "1"}, + {"submit", "Ok"} + }; + if (query.IsImdbQuery) + { + pairs.Add("miben", "imdb"); + pairs.Add("mire", query.ImdbID); + } + else + { + pairs.Add("miben", "name"); + pairs.Add("mire", episodeString == null ? query.GetQueryString() : query.SanitizedSearchTerm); } - foreach (var cat in cats) - pairs.Add(new KeyValuePair("kivalasztott_tipus[]", cat)); - - var results = await PostDataWithCookiesAndRetry(SearchUrl, pairs); - + var cats = MapTorznabCapsToTrackers(query); + if (cats.Count == 0) + cats = GetAllTrackerCategories(); + if (!configData.Hungarian.Value) + cats.RemoveAll(cat => cat.Contains("_hun")); + if (!configData.English.Value) + cats = cats.Except(_languageCats).ToList(); + pairs.Add("kivalasztott_tipus[]", string.Join(",", cats)); + var results = await PostDataWithCookiesAndRetry(SearchUrl, pairs.ToEnumerable(true)); var parser = new HtmlParser(); var dom = parser.ParseDocument(results.Content); - var numVal = 0; // find number of torrents / page - var torrentPerPage = dom.QuerySelector(".box_torrent_all")?.QuerySelectorAll(".box_torrent").Length ?? 0; + var torrentPerPage = dom.QuerySelectorAll(".box_torrent").Length; if (torrentPerPage == 0) return releases; var startPage = (query.Offset / torrentPerPage) + 1; - var previouslyParsedOnPage = query.Offset - (startPage * torrentPerPage) + 1; //+1 because indexing start from 0 - if (previouslyParsedOnPage < 0) - previouslyParsedOnPage = query.Offset; + var previouslyParsedOnPage = query.Offset % torrentPerPage; - // find pagelinks in the bottom - var pageLinks = dom.QuerySelector("div[id=pager_bottom]")?.QuerySelectorAll("a"); - if (pageLinks?.Length > 0) - { - // If there are several pages find the link for the latest one - for (var i = pageLinks.Length - 1; i > 0; i--) - { - var lastPageLink = pageLinks[i].GetAttribute("href").Trim(); - if (lastPageLink.Contains("oldal")) - { - var match = Regex.Match(lastPageLink, @"(?<=oldal=)(\d+)"); - numVal = int.Parse(match.Value); - break; - } - } - } + // find page links in the bottom + var lastPageLink = dom.QuerySelectorAll("div[id=pager_bottom] a[href*=oldal]") + .LastOrDefault()?.GetAttribute("href"); + var pages = int.TryParse(ParseUtil.GetArgumentFromQueryString(lastPageLink, "oldal"), out var lastPage) + ? lastPage + : 1; var limit = query.Limit; if (limit == 0) limit = 100; - if (startPage == 1) { - releases = parseTorrents(results, seasonep, query, releases.Count, limit, previouslyParsedOnPage); + releases = ParseTorrents(results, episodeString, query, releases.Count, limit, previouslyParsedOnPage); previouslyParsedOnPage = 0; startPage++; } - // Check all the pages for the torrents. // The starting index is 2. (the first one is the original where we parse out the pages.) - for (var i = startPage; (i <= numVal && releases.Count < limit); i++) + for (var page = startPage; page <= pages && releases.Count < limit; page++) { - pairs.Add(new KeyValuePair("oldal", i.ToString())); - results = await PostDataWithCookiesAndRetry(SearchUrl, pairs); - releases.AddRange(parseTorrents(results, seasonep, query, releases.Count, limit, previouslyParsedOnPage)); + pairs["oldal"] = page.ToString(); + results = await PostDataWithCookiesAndRetry(SearchUrl, pairs.ToEnumerable(true)); + releases.AddRange(ParseTorrents(results, episodeString, query, releases.Count, limit, previouslyParsedOnPage)); previouslyParsedOnPage = 0; - pairs.Remove(new KeyValuePair("oldal", i.ToString())); } return releases; } - private List parseTorrents(WebClientStringResult results, string seasonep, TorznabQuery query, - int alreadyFounded, int limit, int previouslyParsedOnPage) + private List ParseTorrents(WebClientStringResult results, string episodeString, TorznabQuery query, + int alreadyFound, int limit, int previouslyParsedOnPage) { var releases = new List(); try { var parser = new HtmlParser(); var dom = parser.ParseDocument(results.Content); + var rows = dom.QuerySelectorAll(".box_torrent").Skip(previouslyParsedOnPage).Take(limit - alreadyFound); - var rows = dom.QuerySelector(".box_torrent_all").QuerySelectorAll(".box_torrent"); - + var key = ParseUtil.GetArgumentFromQueryString( + dom.QuerySelector("link[rel=alternate]").GetAttribute("href"), "key"); // Check torrents only till we reach the query Limit - for (var i = previouslyParsedOnPage; (i < rows.Length && ((alreadyFounded + releases.Count) < limit)); i++) - { + foreach (var row in rows) try { - var row = rows[i]; - var key = dom.QuerySelector("link[rel=alternate]").GetAttribute("href").Split('=').Last(); - - var release = new ReleaseInfo(); var torrentTxt = row.QuerySelector(".torrent_txt, .torrent_txt2").QuerySelector("a"); //if (torrentTxt == null) continue; - release.Title = torrentTxt.GetAttribute("title"); - var descr = row.QuerySelector("span")?.GetAttribute("title") + " " + row.QuerySelector("a.infolink")?.TextContent; - release.Description = descr.Trim(); - - release.MinimumRatio = 1; - release.MinimumSeedTime = 172800; // 48 hours - release.DownloadVolumeFactor = 0; - release.UploadVolumeFactor = 1; - + var infoLink = row.QuerySelector("a.infolink"); + var imdbId = ParseUtil.GetLongFromString(infoLink?.GetAttribute("href")); + var desc = row.QuerySelector("span")?.GetAttribute("title") + " " + + infoLink?.TextContent; var downloadLink = SiteLink + torrentTxt.GetAttribute("href"); - var downloadId = downloadLink.Substring(downloadLink.IndexOf("&id=") + 4); + var downloadId = ParseUtil.GetArgumentFromQueryString(downloadLink, "id"); - release.Link = new Uri(SiteLink + "torrents.php?action=download&id=" + downloadId + "&key=" + key); - release.Comments = new Uri(SiteLink + "torrents.php?action=details&id=" + downloadId); - release.Guid = new Uri(release.Comments + "#comments"); + //Build site links + var baseLink = SiteLink + "torrents.php?action=download&id=" + downloadId; + var commentsUri = new Uri(baseLink); + var guidUri = new Uri(baseLink + "#comments"); + var linkUri = new Uri(QueryHelpers.AddQueryString(baseLink, "key", key)); - release.Seeders = ParseUtil.CoerceInt(row.QuerySelector(".box_s2").QuerySelector("a").TextContent); - release.Peers = ParseUtil.CoerceInt(row.QuerySelector(".box_l2").QuerySelector("a").TextContent) + release.Seeders; - var imdblink = row.QuerySelector("a[href*=\".imdb.com/title\"]")?.GetAttribute("href"); - if (!string.IsNullOrWhiteSpace(imdblink)) - release.Imdb = ParseUtil.GetLongFromString(imdblink); + var seeders = ParseUtil.CoerceInt(row.QuerySelector(".box_s2 a").TextContent); + var leechers = ParseUtil.CoerceInt(row.QuerySelector(".box_l2 a").TextContent); + var publishDate = DateTime.Parse( + row.QuerySelector(".box_feltoltve2").InnerHtml.Replace("
", " "), + CultureInfo.InvariantCulture); + var sizeSplit = row.QuerySelector(".box_meret2").TextContent.Split(' '); + var size = ReleaseInfo.GetBytes(sizeSplit[1].ToLower(), ParseUtil.CoerceFloat(sizeSplit[0])); + var catLink = row.QuerySelector("a:has(img[class='categ_link'])").GetAttribute("href"); + var cat = ParseUtil.GetArgumentFromQueryString(catLink, "tipus"); + var title = torrentTxt.GetAttribute("title"); + // if the release name does not contain the language we add from the category + if (cat.Contains("hun") && !title.ToLower().Contains("hun")) + title += ".hun"; + + // Minimum seed time is 48 hours + 24 minutes (.4 hours) per GB of torrent size if downloaded in full. + // Or a 1.0 ratio on the torrent + var seedTime = TimeSpan.FromHours(48) + + TimeSpan.FromMinutes(24 * ReleaseInfo.GigabytesFromBytes(size).Value); + + var release = new ReleaseInfo + { + Title = title, + Description = desc.Trim(), + MinimumRatio = 1, + MinimumSeedTime = (long)seedTime.TotalSeconds, + DownloadVolumeFactor = 0, + UploadVolumeFactor = 1, + Link = linkUri, + Comments = commentsUri, + Guid = guidUri, + Seeders = seeders, + Peers = leechers + seeders, + Imdb = imdbId, + PublishDate = publishDate, + Size = size, + Category = MapTrackerCatToNewznab(cat) + }; var banner = row.QuerySelector("img.infobar_ico")?.GetAttribute("onmouseover"); if (banner != null) { - var bannerRegEx = new Regex(@"mutat\('(.*?)', '", RegexOptions.Compiled); - var bannerMatch = bannerRegEx.Match(banner); - var bannerurl = bannerMatch.Groups[1].Value; - release.BannerUrl = new Uri(bannerurl); + // static call to Regex.Match caches the pattern, so we aren't recompiling every loop. + var bannerMatch = Regex.Match(banner, @"mutat\('(.*?)', '", RegexOptions.Compiled); + release.BannerUrl = new Uri(bannerMatch.Groups[1].Value); } - release.PublishDate = DateTime.Parse(row.QuerySelector(".box_feltoltve2").InnerHtml.Replace("
", " "), CultureInfo.InvariantCulture); - var sizeSplit = row.QuerySelector(".box_meret2").TextContent.Split(' '); - release.Size = ReleaseInfo.GetBytes(sizeSplit[1].ToLower(), ParseUtil.CoerceFloat(sizeSplit[0])); - var catlink = row.QuerySelector("a:has(img[class='categ_link'])").GetAttribute("href"); - var cat = ParseUtil.GetArgumentFromQueryString(catlink, "tipus"); - release.Category = MapTrackerCatToNewznab(cat); - /* if the release name not contains the language we add it because it is know from category */ - if (cat.Contains("hun") && !release.Title.ToLower().Contains("hun")) - release.Title += ".hun"; - - if (seasonep == null) - releases.Add(release); - - else + //TODO there is room for improvement here. + if (episodeString != null && + query.MatchQueryStringAND(release.Title, queryStringOverride: episodeString) && + !query.IsImdbQuery) { - if (query.MatchQueryStringAND(release.Title, null, seasonep)) + // For Sonarr if the search query was english the title must be english also + // The description holds the alternate language name + // so we need to swap title and description names + var tempTitle = release.Title; + + // releaseData everything after Name.S0Xe0X + var releaseIndex = tempTitle.IndexOf(episodeString, StringComparison.OrdinalIgnoreCase) + + episodeString.Length; + var releaseData = tempTitle.Substring(releaseIndex).Trim(); + + // release description contains [imdb: ****] but we only need the data before it for title + var description = new[] { - /* For sonnar if the search querry was english the title must be english also so we need to change the Description and Title */ - var temp = release.Title; - - // releasedata everithing after Name.S0Xe0X - var releasedata = release.Title.Split(new[] { seasonep }, StringSplitOptions.None)[1].Trim(); - - /* if the release name not contains the language we add it because it is know from category */ - if (cat.Contains("hun") && !releasedata.Contains("hun")) - releasedata += ".hun"; - - // release description contains [imdb: ****] but we only need the data before it for title - string[] description = { release.Description, "" }; - if (release.Description.Contains("[imdb:")) - { - description = release.Description.Split('['); - description[1] = "[" + description[1]; - } - - release.Title = (description[0].Trim() + "." + seasonep.Trim() + "." + releasedata.Trim('.')).Replace(' ', '.'); - - // if search is done for S0X than we dont want to put . between S0X and E0X - var match = Regex.Match(releasedata, @"^E\d\d?"); - if (seasonep.Length == 3 && match.Success) - release.Title = (description[0].Trim() + "." + seasonep.Trim() + releasedata.Trim('.')).Replace(' ', '.'); - - // add back imdb points to the description [imdb: 8.7] - release.Description = temp + " " + description[1]; - release.Description = release.Description.Trim(); - releases.Add(release); + release.Description, + "" + }; + if (release.Description.Contains("[imdb:")) + { + description = release.Description.Split('['); + description[1] = "[" + description[1]; } + + var match = Regex.Match(releaseData, @"^E\d\d?"); + // if search is done for S0X than we don't want to put . between S0X and E0X + var episodeSeparator = episodeString.Length == 3 && match.Success ? null : "."; + release.Title = + (description[0].Trim() + "." + episodeString.Trim() + episodeSeparator + + releaseData.Trim('.')).Replace(' ', '.'); + + // add back imdb points to the description [imdb: 8.7] + release.Description = tempTitle + " " + description[1]; + release.Description = release.Description.Trim(); } + + releases.Add(release); } catch (FormatException ex) { - logger.Error("Problem of parsing Torrent:" + rows[i].InnerHtml); + logger.Error("Problem of parsing Torrent:" + row.InnerHtml); logger.Error("Exception was the following:" + ex); } - } } catch (Exception ex) { diff --git a/src/Jackett.Common/Models/ReleaseInfo.cs b/src/Jackett.Common/Models/ReleaseInfo.cs index 2b411b508..fa8bcc087 100644 --- a/src/Jackett.Common/Models/ReleaseInfo.cs +++ b/src/Jackett.Common/Models/ReleaseInfo.cs @@ -37,7 +37,7 @@ namespace Jackett.Common.Models public IIndexer Origin; - private static double? GigabytesFromBytes(double? size) => size / 1024.0 / 1024.0 / 1024.0; + public static double? GigabytesFromBytes(double? size) => size / 1024.0 / 1024.0 / 1024.0; public double? Gain => Seeders * GigabytesFromBytes(Size); public ReleaseInfo() diff --git a/src/Jackett.Test/Jackett.Test.csproj b/src/Jackett.Test/Jackett.Test.csproj index 4aeb1f4b7..0575f9da7 100644 --- a/src/Jackett.Test/Jackett.Test.csproj +++ b/src/Jackett.Test/Jackett.Test.csproj @@ -1,4 +1,4 @@ - + netcoreapp3.1;net461