From ab9d8f640342be4ef03587304b934a63921f1ed6 Mon Sep 17 00:00:00 2001 From: flightlevel Date: Thu, 10 Jan 2019 20:12:57 +1100 Subject: [PATCH 1/5] Package update --- src/Jackett.Common/Jackett.Common.csproj | 2 +- src/Jackett.Server/Jackett.Server.csproj | 6 +++--- src/Jackett.Tray/Jackett.Tray.csproj | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Jackett.Common/Jackett.Common.csproj b/src/Jackett.Common/Jackett.Common.csproj index 0f329571d..140fa9441 100644 --- a/src/Jackett.Common/Jackett.Common.csproj +++ b/src/Jackett.Common/Jackett.Common.csproj @@ -12,7 +12,7 @@ - + diff --git a/src/Jackett.Server/Jackett.Server.csproj b/src/Jackett.Server/Jackett.Server.csproj index a1a9f863a..c4d9f9212 100644 --- a/src/Jackett.Server/Jackett.Server.csproj +++ b/src/Jackett.Server/Jackett.Server.csproj @@ -25,7 +25,7 @@ - + @@ -35,9 +35,9 @@ - + - + diff --git a/src/Jackett.Tray/Jackett.Tray.csproj b/src/Jackett.Tray/Jackett.Tray.csproj index 7bfceafd1..519b10e34 100644 --- a/src/Jackett.Tray/Jackett.Tray.csproj +++ b/src/Jackett.Tray/Jackett.Tray.csproj @@ -110,7 +110,7 @@ - 2.3.0 + 2.4.3 From e188454fec26ffd674c487650f397de8986c7b77 Mon Sep 17 00:00:00 2001 From: flightlevel Date: Thu, 10 Jan 2019 20:22:15 +1100 Subject: [PATCH 2/5] Use NETStandard for Jackett.Common --- src/Jackett.Common/Jackett.Common.csproj | 19 ++++--------------- src/Jackett.Updater/Jackett.Updater.csproj | 2 +- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/Jackett.Common/Jackett.Common.csproj b/src/Jackett.Common/Jackett.Common.csproj index 140fa9441..aab0f03de 100644 --- a/src/Jackett.Common/Jackett.Common.csproj +++ b/src/Jackett.Common/Jackett.Common.csproj @@ -1,7 +1,7 @@  - netstandard2.0;net461 + netstandard2.0 0.0.0 NU1605 @@ -13,28 +13,17 @@ + + - - - - - - - - - - - - - - + diff --git a/src/Jackett.Updater/Jackett.Updater.csproj b/src/Jackett.Updater/Jackett.Updater.csproj index cae4a0d6a..df289bd93 100644 --- a/src/Jackett.Updater/Jackett.Updater.csproj +++ b/src/Jackett.Updater/Jackett.Updater.csproj @@ -1,7 +1,7 @@  - net461;netcoreapp2.1 + net461;netcoreapp2.2 jackett.ico JackettUpdater Exe From cb8835b878a1a3c63035923306979aee6eb7eadd Mon Sep 17 00:00:00 2001 From: snamds Date: Thu, 10 Jan 2019 15:53:15 +0100 Subject: [PATCH 3/5] Newpct: fix Nan size parse (#4428) --- src/Jackett.Common/Indexers/Newpct.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Jackett.Common/Indexers/Newpct.cs b/src/Jackett.Common/Indexers/Newpct.cs index c701478f5..e6b6e4bf5 100644 --- a/src/Jackett.Common/Indexers/Newpct.cs +++ b/src/Jackett.Common/Indexers/Newpct.cs @@ -543,8 +543,15 @@ namespace Jackett.Common.Indexers var pubDateText = span[1].TextContent.Trim(); var sizeText = span[2].TextContent.Trim(); - long size = ReleaseInfo.GetBytes(sizeText); - DateTime publishDate = DateTime.ParseExact(pubDateText, "dd-MM-yyyy", null); + long size = 0; + try + { + size = ReleaseInfo.GetBytes(sizeText); + } catch + { + } + DateTime publishDate; + DateTime.TryParseExact(pubDateText, "dd-MM-yyyy", null, DateTimeStyles.None, out publishDate); var div = row.QuerySelector("div"); @@ -654,7 +661,8 @@ namespace Jackett.Common.Indexers result.Category = new List { TorznabCatType.Movies.ID }; } - result.Size = size; + if (size > 0) + result.Size = size; result.Link = new Uri(detailsUrl); result.Guid = result.Link; result.PublishDate = publishDate; From d878c9e05effa7bd20ff3915d823650dc70af982 Mon Sep 17 00:00:00 2001 From: CodeMonkey Date: Thu, 10 Jan 2019 09:53:55 -0500 Subject: [PATCH 4/5] HorribleSubs: Fix the Indexer (#4429) - Now returns batches in results along with individual episodes - Now returns all episodes for a show instead of just the first page - No longer makes a bunch of unnecessary API calls (faster results) --- src/Jackett.Common/Indexers/HorribleSubs.cs | 150 +++++++++++--------- 1 file changed, 82 insertions(+), 68 deletions(-) diff --git a/src/Jackett.Common/Indexers/HorribleSubs.cs b/src/Jackett.Common/Indexers/HorribleSubs.cs index e37374ff5..01128586d 100644 --- a/src/Jackett.Common/Indexers/HorribleSubs.cs +++ b/src/Jackett.Common/Indexers/HorribleSubs.cs @@ -46,7 +46,7 @@ namespace Jackett.Common.Indexers configData.LoadValuesFromJson(configJson); var releases = await PerformQuery(new TorznabQuery()); - await ConfigureIfOK(string.Empty, releases.Count() > 0, () => + await ConfigureIfOK(string.Empty, releases.Any(), () => { throw new Exception("Could not find releases from this URL"); }); @@ -56,10 +56,57 @@ namespace Jackett.Common.Indexers protected override async Task> PerformQuery(TorznabQuery query) { - return await PerformQuery(query, 0); + var ResultParser = new HtmlParser(); + var releases = new List(); + var searchString = query.GetQueryString(); + var queryCollection = new NameValueCollection(); + + + if (string.IsNullOrWhiteSpace(searchString)) + { + return await PerformLatestQuery(query); + } + else + { + queryCollection.Add("method", "search"); + + searchString = searchString.Replace("'", ""); // ignore ' (e.g. search for america's Next Top Model) + queryCollection.Add("value", searchString); + } + + var searchUrl = ApiEndpoint + "?" + queryCollection.GetQueryString(); + var response = await RequestStringWithCookiesAndRetry(searchUrl, string.Empty); + + try + { + if (response.Content.Contains("Nothing was found")) + { + return releases.ToArray(); + } + var dom = ResultParser.Parse(response.Content); + var resultLinks = dom.QuerySelectorAll("ul > li > a"); + var uniqueShowLinks = new HashSet(); + foreach (var resultLink in resultLinks) + { + var href = SiteLink + resultLink.Attributes["href"].Value.Substring(1); // = https://horriblesubs.info/shows/boruto-naruto-next-generations#71 + var showUrl = href.Substring(0, href.LastIndexOf("#")); + uniqueShowLinks.Add(showUrl); + } + foreach (var showLink in uniqueShowLinks) + { + var showReleases = await GetReleases(showLink, latestOnly: false); + releases.AddRange(showReleases); + } + } + catch (Exception ex) + { + OnParseError(response.Content, ex); + } + + return releases; } - private async Task> PerformLatestQuery(TorznabQuery query, int attempts) + private async Task> PerformLatestQuery(TorznabQuery query) { var ResultParser = new HtmlParser(); var releases = new List(); @@ -80,13 +127,14 @@ namespace Jackett.Common.Indexers var dom = ResultParser.Parse(response.Content); var latestresults = dom.QuerySelectorAll("ul > li > a"); - foreach (var row in latestresults) + foreach (var resultLink in latestresults) { - var href = SiteLink + row.Attributes["href"].Value.Substring(1); - var showrels = await GetRelease(href); - releases.AddRange(showrels); + var href = SiteLink + resultLink.Attributes["href"].Value.Substring(1); // = https://horriblesubs.info/shows/boruto-naruto-next-generations#71 + var episodeNumber = href.Substring(href.LastIndexOf("#") + 1); // = 71 + var showUrl = href.Substring(0, href.LastIndexOf("#")); + var showReleases = await GetReleases(showUrl, latestOnly: true, titleContains: episodeNumber); + releases.AddRange(showReleases); } - } catch (Exception ex) { @@ -96,15 +144,12 @@ namespace Jackett.Common.Indexers return releases; } - private async Task> GetRelease(string ResultURL) + private async Task> GetReleases(string ResultURL, bool latestOnly, string titleContains = null) { var releases = new List(); var ResultParser = new HtmlParser(); try { - var episodeno = ResultURL.Substring(ResultURL.LastIndexOf("#") + 1); // = 71 - ResultURL = ResultURL.Replace("#" + episodeno, ""); // = https://horriblesubs.info/shows/boruto-naruto-next-generations - var showPageResponse = await RequestStringWithCookiesAndRetry(ResultURL, string.Empty); await FollowIfRedirect(showPageResponse); @@ -116,12 +161,28 @@ namespace Jackett.Common.Indexers int ShowID = int.Parse(match.Groups[2].Value); - string showAPIURL = ApiEndpoint + "?method=getshows&type=show&showid=" + ShowID; //https://horriblesubs.info/api.php?method=getshows&type=show&showid=869 - var showAPIResponse = await RequestStringWithCookiesAndRetry(showAPIURL, string.Empty); + var apiUrls = new string[] { + ApiEndpoint + "?method=getshows&type=batch&showid=" + ShowID, //https://horriblesubs.info/api.php?method=getshows&type=batch&showid=1194 + ApiEndpoint + "?method=getshows&type=show&showid=" + ShowID //https://horriblesubs.info/api.php?method=getshows&type=show&showid=869 + }; + var releaserows = new List(); + foreach (string apiUrl in apiUrls) + { + int nextId = 0; + while(true) + { + var showAPIResponse = await RequestStringWithCookiesAndRetry(apiUrl + "&nextid=" + nextId, string.Empty); + var showAPIdom = ResultParser.Parse(showAPIResponse.Content); + var releaseRowResults = showAPIdom.QuerySelectorAll("div.rls-info-container"); + releaserows.AddRange(releaseRowResults); + nextId++; - var showAPIdom = ResultParser.Parse(showAPIResponse.Content); - var releaserows = showAPIdom.QuerySelectorAll("div.rls-info-container"); + if (releaseRowResults.Length == 0 || latestOnly) { + break; + } + } + } foreach (var releaserow in releaserows) { @@ -130,14 +191,14 @@ namespace Jackett.Common.Indexers title = title.Replace("SD720p1080p", ""); title = title.Replace(dateStr, ""); - if (title.Contains(episodeno) == false) + if (!string.IsNullOrWhiteSpace(titleContains) && !title.Contains(titleContains)) { continue; } // Ensure fansub group name is present in the title // This is needed for things like configuring tag restrictions in Sonarr - if (title.Contains("[HorribleSubs]") == false) + if (!title.Contains("[HorribleSubs]")) { title = "[HorribleSubs] " + title; } @@ -175,6 +236,7 @@ namespace Jackett.Common.Indexers if (p480.QuerySelector(".hs-torrent-link > a") != null) { release.Link = new Uri(p480.QuerySelector(".hs-torrent-link > a").GetAttribute("href")); + release.Comments = new Uri(release.Link.AbsoluteUri.Replace("/torrent", string.Empty)); release.Guid = release.Link; } if (p480.QuerySelector(".hs-magnet-link > a") != null) @@ -204,6 +266,7 @@ namespace Jackett.Common.Indexers if (p720.QuerySelector(".hs-torrent-link > a") != null) { release.Link = new Uri(p720.QuerySelector(".hs-torrent-link > a").GetAttribute("href")); + release.Comments = new Uri(release.Link.AbsoluteUri.Replace("/torrent", string.Empty)); release.Guid = release.Link; } if (p720.QuerySelector(".hs-magnet-link > a") != null) @@ -233,6 +296,7 @@ namespace Jackett.Common.Indexers if (p1080.QuerySelector(".hs-torrent-link > a") != null) { release.Link = new Uri(p1080.QuerySelector(".hs-torrent-link > a").GetAttribute("href")); + release.Comments = new Uri(release.Link.AbsoluteUri.Replace("/torrent", string.Empty)); release.Guid = release.Link; } if (p1080.QuerySelector(".hs-magnet-link > a") != null) @@ -250,55 +314,5 @@ namespace Jackett.Common.Indexers } return releases; } - - public async Task> PerformQuery(TorznabQuery query, int attempts) - { - var ResultParser = new HtmlParser(); - var releases = new List(); - var searchString = query.GetQueryString(); - var queryCollection = new NameValueCollection(); - - - if (string.IsNullOrWhiteSpace(searchString)) - { - return await PerformLatestQuery(query, attempts); - } - else - { - queryCollection.Add("method", "search"); - - searchString = searchString.Replace("'", ""); // ignore ' (e.g. search for america's Next Top Model) - queryCollection.Add("value", searchString); - } - - var searchUrl = ApiEndpoint + "?" + queryCollection.GetQueryString(); - var response = await RequestStringWithCookiesAndRetry(searchUrl, string.Empty); - - try - { - if (response.Content.Contains("Nothing was found")) - { - return releases.ToArray(); - } - var dom = ResultParser.Parse(response.Content); - var showlinks = dom.QuerySelectorAll("ul > li > a"); - foreach (var showlink in showlinks) - { - var href = SiteLink + showlink.Attributes["href"].Value.Substring(1); // = https://horriblesubs.info/shows/boruto-naruto-next-generations#71 - - var showrels = await GetRelease(href); - releases.AddRange(showrels); - - - } - } - catch (Exception ex) - { - OnParseError(response.Content, ex); - } - - return releases; - } - } } From 6b536ce62f9be0ab9b7f77eb0b936e6d69769776 Mon Sep 17 00:00:00 2001 From: Mystere98 <38164068+Mystere98@users.noreply.github.com> Date: Thu, 10 Jan 2019 15:54:28 +0100 Subject: [PATCH 5/5] Xthor: Update Cats (#4430) --- src/Jackett.Common/Indexers/Xthor.cs | 114 +++++++++++++++------------ 1 file changed, 62 insertions(+), 52 deletions(-) diff --git a/src/Jackett.Common/Indexers/Xthor.cs b/src/Jackett.Common/Indexers/Xthor.cs index fe4b48411..32a5209e9 100644 --- a/src/Jackett.Common/Indexers/Xthor.cs +++ b/src/Jackett.Common/Indexers/Xthor.cs @@ -58,63 +58,73 @@ namespace Jackett.Common.Indexers TorznabCaps.Categories.Clear(); // Movies - AddCategoryMapping(6, TorznabCatType.MoviesSD, "XVID"); - AddCategoryMapping(7, TorznabCatType.MoviesSD, "X264"); - AddCategoryMapping(95, TorznabCatType.MoviesSD, "WEBRIP"); - AddCategoryMapping(5, TorznabCatType.MoviesHD, "HD 720P"); - AddCategoryMapping(4, TorznabCatType.MoviesHD, "HD 1080P X264"); - AddCategoryMapping(100, TorznabCatType.MoviesHD, "HD 1080P X265"); - AddCategoryMapping(94, TorznabCatType.MoviesHD, "WEBDL"); - AddCategoryMapping(107, TorznabCatType.MoviesHD, "4K"); - AddCategoryMapping(1, TorznabCatType.MoviesBluRay, "FULL BLURAY"); - AddCategoryMapping(2, TorznabCatType.MoviesBluRay, "BLURAY REMUX"); - AddCategoryMapping(3, TorznabCatType.MoviesBluRay, "FULL BLURAY 3D"); - AddCategoryMapping(8, TorznabCatType.MoviesDVD, "FULL DVD"); - AddCategoryMapping(9, TorznabCatType.MoviesOther, "VOSTFR"); - AddCategoryMapping(36, TorznabCatType.XXX, "XXX"); - + AddCategoryMapping(118, TorznabCatType.MoviesBluRay, "UHD FULL BLURAY"); + AddCategoryMapping(119, TorznabCatType.MoviesBluRay, "UHD BLURAY REMUX"); + AddCategoryMapping(107, TorznabCatType.MoviesUHD, "UHD 2160P X265"); + AddCategoryMapping(1, TorznabCatType.MoviesBluRay, "FULL BLURAY"); + AddCategoryMapping(2, TorznabCatType.MoviesBluRay, "BLURAY REMUX"); + AddCategoryMapping(100, TorznabCatType.MoviesHD, "HD 1080P X265"); + AddCategoryMapping(4, TorznabCatType.MoviesHD, "HD 1080P X264"); + AddCategoryMapping(5, TorznabCatType.MoviesHD, "HD 720P X264"); + AddCategoryMapping(7, TorznabCatType.MoviesSD, "SD X264"); + AddCategoryMapping(8, TorznabCatType.MoviesDVD, "FULL DVD"); + AddCategoryMapping(3, TorznabCatType.Movies3D, "3D"); + AddCategoryMapping(6, TorznabCatType.MoviesSD, "XVID"); + AddCategoryMapping(122, TorznabCatType.MoviesHD, "HDTV"); + AddCategoryMapping(94, TorznabCatType.MoviesWEBDL, "WEBDL"); + AddCategoryMapping(95, TorznabCatType.MoviesWEBDL, "WEBRIP"); + AddCategoryMapping(12, TorznabCatType.TVDocumentary, "DOCS"); + AddCategoryMapping(33, TorznabCatType.MoviesOther, "SPECTACLE"); + AddCategoryMapping(31, TorznabCatType.MoviesOther, "ANIMATION"); + AddCategoryMapping(9, TorznabCatType.MoviesOther, "VOSTFR"); + // Series - AddCategoryMapping(14, TorznabCatType.TVSD, "SD VF"); - AddCategoryMapping(16, TorznabCatType.TVSD, "SD VF VOSTFR"); - AddCategoryMapping(15, TorznabCatType.TVHD, "HD VF"); - AddCategoryMapping(17, TorznabCatType.TVHD, "HD VF VOSTFR"); - AddCategoryMapping(13, TorznabCatType.TVOTHER, "PACK"); - AddCategoryMapping(98, TorznabCatType.TVOTHER, "PACK VOSTFR HD"); - AddCategoryMapping(16, TorznabCatType.TVOTHER, "PACK VOSTFR SD"); - AddCategoryMapping(30, TorznabCatType.TVOTHER, "EMISSIONS"); - AddCategoryMapping(34, TorznabCatType.TVOTHER, "EMISSIONS"); - AddCategoryMapping(33, TorznabCatType.TVOTHER, "SHOWS"); - - // Anime - AddCategoryMapping(31, TorznabCatType.TVAnime, "MOVIES ANIME"); - AddCategoryMapping(32, TorznabCatType.TVAnime, "SERIES ANIME"); - AddCategoryMapping(110, TorznabCatType.TVAnime, "ANIME VOSTFR"); - AddCategoryMapping(101, TorznabCatType.TVAnime, "PACK ANIME"); - - // Documentaries - AddCategoryMapping(12, TorznabCatType.TVDocumentary, "DOCS"); + AddCategoryMapping(104, TorznabCatType.TVOTHER, "BLURAY"); + AddCategoryMapping(13, TorznabCatType.TVOTHER, "PACK VF"); + AddCategoryMapping(15, TorznabCatType.TVHD, "HD VF"); + AddCategoryMapping(14, TorznabCatType.TVSD, "SD VF"); + AddCategoryMapping(98, TorznabCatType.TVOTHER, "PACK VOSTFR"); + AddCategoryMapping(17, TorznabCatType.TVHD, "HD VF VOSTFR"); + AddCategoryMapping(16, TorznabCatType.TVSD, "SD VF VOSTFR"); + AddCategoryMapping(101, TorznabCatType.TVAnime, "PACK ANIME"); + AddCategoryMapping(32, TorznabCatType.TVAnime, "ANIME VF"); + AddCategoryMapping(110, TorznabCatType.TVAnime, "ANIME VOSTFR"); + AddCategoryMapping(123, TorznabCatType.TVOTHER, "ANIMATION"); + AddCategoryMapping(109, TorznabCatType.TVDocumentary, "DOCS"); + AddCategoryMapping(30, TorznabCatType.TVOTHER, "EMISSIONS"); + AddCategoryMapping(34, TorznabCatType.TVOTHER, "SPORT"); // Music - AddCategoryMapping(20, TorznabCatType.AudioVideo, "CONCERT"); + AddCategoryMapping(20, TorznabCatType.AudioVideo, "CONCERT"); + + // Books + AddCategoryMapping(24, TorznabCatType.BooksEbook, "ENOOKS NOVEL"); + AddCategoryMapping(96, TorznabCatType.BooksMagazines, "EBOOKS MAGAZINES"); + AddCategoryMapping(116, TorznabCatType.BooksEbook, "EBOOKS NOVEL JUNIOR"); + AddCategoryMapping(99, TorznabCatType.BooksOther, "EBOOKS BD"); + AddCategoryMapping(102, TorznabCatType.BooksComics, "EBOOKS COMICS"); + AddCategoryMapping(103, TorznabCatType.BooksOther, "EBOOKS MANGA"); - // Other - AddCategoryMapping(21, TorznabCatType.PC, "PC"); - AddCategoryMapping(22, TorznabCatType.PCMac, "PC"); - AddCategoryMapping(25, TorznabCatType.PCGames, "GAMES"); - AddCategoryMapping(26, TorznabCatType.ConsoleXbox360, "GAMES"); - AddCategoryMapping(28, TorznabCatType.ConsoleWii, "GAMES"); - AddCategoryMapping(27, TorznabCatType.ConsolePS3, "GAMES"); - AddCategoryMapping(29, TorznabCatType.ConsoleNDS, "GAMES"); - AddCategoryMapping(24, TorznabCatType.BooksEbook, "EBOOKS"); - AddCategoryMapping(96, TorznabCatType.BooksEbook, "EBOOKS MAGAZINES"); - AddCategoryMapping(99, TorznabCatType.BooksEbook, "EBOOKS ANIME"); - AddCategoryMapping(23, TorznabCatType.PCPhoneAndroid, "ANDROID"); + // SOFTWARE + AddCategoryMapping(25, TorznabCatType.PCGames, "PC GAMES"); + AddCategoryMapping(27, TorznabCatType.ConsolePS3, "PS GAMES"); + AddCategoryMapping(111, TorznabCatType.PCMac, "MAC GAMES"); + AddCategoryMapping(112, TorznabCatType.PC, "LINUX GAMES"); + AddCategoryMapping(26, TorznabCatType.ConsoleXbox360, "XBOX GAMES"); + AddCategoryMapping(28, TorznabCatType.ConsoleWii, "WII GAMES"); + AddCategoryMapping(29, TorznabCatType.ConsoleNDS, "NDS GAMES"); + AddCategoryMapping(117, TorznabCatType.PC, "ROM"); + AddCategoryMapping(21, TorznabCatType.PC, "PC SOFTWARE"); + AddCategoryMapping(22, TorznabCatType.PCMac, "MAC SOFTWARE"); + AddCategoryMapping(23, TorznabCatType.PCPhoneAndroid, "ANDROID"); - AddCategoryMapping(36, TorznabCatType.XXX, "XxX / Films"); - AddCategoryMapping(105, TorznabCatType.XXX, "XxX / Séries"); - AddCategoryMapping(114, TorznabCatType.XXX, "XxX / Lesbiennes "); - AddCategoryMapping(115, TorznabCatType.XXX, "XxX / Gays"); - AddCategoryMapping(113, TorznabCatType.XXX, "XxX / Hentai"); + // XxX + AddCategoryMapping(36, TorznabCatType.XXX, "XxX / Films"); + AddCategoryMapping(105, TorznabCatType.XXX, "XxX / Séries"); + AddCategoryMapping(114, TorznabCatType.XXX, "XxX / Lesbiennes"); + AddCategoryMapping(115, TorznabCatType.XXX, "XxX / Gays"); + AddCategoryMapping(113, TorznabCatType.XXX, "XxX / Hentai"); + AddCategoryMapping(120, TorznabCatType.XXX, "XxX / Magazines"); } ///